Compare commits
48
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
eea96e8674 | ||
|
|
4c5583e725 | ||
|
|
441ad0b18e | ||
|
|
792ff9d1ee | ||
|
|
af49e94d97 | ||
|
|
37c00b609c | ||
|
|
94a3a762b0 | ||
|
|
9a7d678532 | ||
|
|
823d42d528 | ||
|
|
de514115dd | ||
|
|
0f8816f659 | ||
|
|
b59eebf1e7 | ||
|
|
f4061f542c | ||
|
|
e99f84bd01 | ||
|
|
9653219c53 | ||
|
|
f44382fb09 | ||
|
|
dd73e0ad74 | ||
|
|
7f045c626a | ||
|
|
e99d26de89 | ||
|
|
7f22006e97 | ||
|
|
e35fc3e6d6 | ||
|
|
5f52dad5f1 | ||
|
|
15064d6fd5 | ||
|
|
e0060c9e6e | ||
|
|
a3a36cd5d7 | ||
|
|
8afaee7d21 | ||
|
|
c8280de9c3 | ||
|
|
f8b9df6438 | ||
|
|
92d14fbd60 | ||
|
|
01f6b99631 | ||
|
|
8d5e4ee052 | ||
|
|
213c7f0362 | ||
|
|
9e0aab6b6a | ||
|
|
3eb5a454fd | ||
|
|
dc49bf4d14 | ||
|
|
f7fb115a0f | ||
|
|
3199a6f1fb | ||
|
|
2b45a2e412 | ||
|
|
3fadf82909 | ||
|
|
2a851ea230 | ||
|
|
0502eb45ed | ||
|
|
11ba361c8c | ||
|
|
ba75ab4ecc | ||
|
|
afffa0fc96 | ||
|
|
32b22d0828 | ||
|
|
ac3a63973d | ||
|
|
e61a475859 | ||
|
|
beb6c33e63 |
@@ -44,6 +44,10 @@ jobs:
|
||||
# schema has, and name nothing that is gone.
|
||||
- if: always()
|
||||
run: python3 tools/fork/privacy-check.py
|
||||
# The admin reads each expression field's allowed values and variables
|
||||
# from the schema; they're generated from the registry and must match it.
|
||||
- if: always()
|
||||
run: python3 tools/fork/expr-schema.py --check
|
||||
- if: always()
|
||||
run: python3 -m unittest discover -s tools/fork/tests
|
||||
|
||||
|
||||
Generated
+6
@@ -3947,9 +3947,14 @@ name = "inbuxa-features"
|
||||
version = "0.16.22"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"aho-corasick",
|
||||
"base64 0.23.1",
|
||||
"flate2",
|
||||
"jmap_proto",
|
||||
"mail-builder 1.0.0",
|
||||
"mail-parser",
|
||||
"quick-xml 0.41.0",
|
||||
"regex",
|
||||
"registry",
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -3961,6 +3966,7 @@ dependencies = [
|
||||
"types",
|
||||
"utils",
|
||||
"xxhash-rust",
|
||||
"zip",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -165,6 +165,14 @@ impl AccessToken {
|
||||
mut requested_permissions: Permissions,
|
||||
) -> Result<(), Vec<Permission>> {
|
||||
requested_permissions.difference(self.permissions_bits());
|
||||
// inbuxa: journaling, JR-18: whoever sets up journals may give
|
||||
// others (or, through a role, themselves) the reading of them,
|
||||
// which administrators don't hold by default; the role change is
|
||||
// in the audit log
|
||||
if self.has_permission(Permission::SysJournalUpdate) {
|
||||
requested_permissions.clear(Permission::SysJournalSearch as usize);
|
||||
requested_permissions.clear(Permission::SysJournalExport as usize);
|
||||
}
|
||||
if requested_permissions.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
@@ -296,6 +304,27 @@ impl Default for DefaultPermissions {
|
||||
default.superuser.push(permission);
|
||||
default.tenant.push(permission);
|
||||
}
|
||||
// inbuxa: DLP and mail flow rules, and held mail, are the
|
||||
// server's: never a tenant's (dlp-and-mail-flow-rules spec,
|
||||
// settled answer 3)
|
||||
Permission::SysMailRuleGet
|
||||
| Permission::SysMailRuleUpdate
|
||||
| Permission::SysDlpPolicyGet
|
||||
| Permission::SysDlpPolicyUpdate
|
||||
| Permission::SysDlpReviewGet
|
||||
| Permission::SysDlpReviewUpdate
|
||||
// inbuxa: every security check is server-wide (security
|
||||
// to-do list spec)
|
||||
| Permission::SysSecurityAccept => {
|
||||
default.superuser.push(permission);
|
||||
}
|
||||
// inbuxa: journals are the server's; administrators set them
|
||||
// up but read what's journaled only if granted it
|
||||
// (journaling spec, JR-18, settled answer 5)
|
||||
Permission::SysJournalGet | Permission::SysJournalUpdate => {
|
||||
default.superuser.push(permission);
|
||||
}
|
||||
Permission::SysJournalSearch | Permission::SysJournalExport => {}
|
||||
// inbuxa: AL-12: tenant administrators lock and delegate
|
||||
// within their tenant
|
||||
Permission::SysAccountLockGet
|
||||
|
||||
@@ -70,6 +70,7 @@ pub mod cache;
|
||||
pub mod audit; // inbuxa: the audit log (audit-hold-lock spec, AU)
|
||||
pub mod hold; // inbuxa: legal holds (audit-hold-lock spec, LH)
|
||||
pub mod privacy; // inbuxa: the personal-data catalog, evaluated
|
||||
pub mod reachability; // inbuxa: whether the outside world reaches each node's ports
|
||||
pub mod config;
|
||||
pub mod expr;
|
||||
pub mod i18n;
|
||||
@@ -129,6 +130,8 @@ pub const KV_LOCK_QUEUE_MESSAGE: u8 = 21;
|
||||
pub const KV_LOCK_TASK: u8 = 23;
|
||||
pub const KV_LOCK_DAV: u8 = 25;
|
||||
pub const KV_SIEVE_ID: u8 = 26;
|
||||
// inbuxa: far above upstream's prefixes, so a new one of theirs never collides
|
||||
pub const KV_PORT_REACHABILITY: u8 = 200;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Server {
|
||||
|
||||
@@ -65,6 +65,14 @@ const OFFICER: &[Permission] = &[
|
||||
Permission::SysLegalHoldUpdate,
|
||||
Permission::SysLegalHoldExport,
|
||||
Permission::SysAccountLockGet,
|
||||
// dlp-and-mail-flow-rules spec, §2.8: see DLP rules, review held mail
|
||||
Permission::SysDlpPolicyGet,
|
||||
Permission::SysDlpReviewGet,
|
||||
Permission::SysDlpReviewUpdate,
|
||||
// journaling spec, JR-18: see journals, search and export them
|
||||
Permission::SysJournalGet,
|
||||
Permission::SysJournalSearch,
|
||||
Permission::SysJournalExport,
|
||||
];
|
||||
|
||||
/// What a tenant's officer holds besides [`READS`].
|
||||
@@ -113,6 +121,11 @@ fn created_key(tenant: Option<Id>) -> ValueClass {
|
||||
})
|
||||
}
|
||||
|
||||
/// The server-level Compliance Officer role the server made, if it has.
|
||||
pub async fn server_role(data: &Store) -> trc::Result<Option<Id>> {
|
||||
recorded(data, None).await
|
||||
}
|
||||
|
||||
async fn recorded(data: &Store, tenant: Option<Id>) -> trc::Result<Option<Id>> {
|
||||
Ok(data
|
||||
.get_value::<u64>(ValueKey::from(created_key(tenant)))
|
||||
@@ -172,13 +185,19 @@ pub async fn ensure_compliance_roles(registry: &RegistryStore, data: &Store) ->
|
||||
|
||||
/// A new tenant gets its Compliance Officer role.
|
||||
pub async fn tenant_created(registry: &RegistryStore, data: &Store, tenant: Id) -> trc::Result<()> {
|
||||
create_once(registry, data, Some(tenant), tenant_role(tenant)).await.map(|_| ())
|
||||
create_once(registry, data, Some(tenant), tenant_role(tenant))
|
||||
.await
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
/// Before a tenant is deleted: removes its Compliance Officer role if nobody
|
||||
/// holds it, so the role doesn't block the delete. Returns whether it did,
|
||||
/// so a delete refused for another reason can put it back.
|
||||
pub async fn tenant_deleting(registry: &RegistryStore, data: &Store, tenant: Id) -> trc::Result<bool> {
|
||||
pub async fn tenant_deleting(
|
||||
registry: &RegistryStore,
|
||||
data: &Store,
|
||||
tenant: Id,
|
||||
) -> trc::Result<bool> {
|
||||
let Some(role) = recorded(data, Some(tenant)).await? else {
|
||||
return Ok(false);
|
||||
};
|
||||
@@ -220,7 +239,9 @@ mod tests {
|
||||
// Beyond what any user holds for their own account
|
||||
for permission in all.into_iter().filter(|p| !user.contains(p)) {
|
||||
let name = permission.as_str();
|
||||
let holds = name.starts_with("sysLegalHold");
|
||||
// Placing holds and reviewing held mail are the officer's
|
||||
// job, not settings (settled answers 2 and 4)
|
||||
let holds = name.starts_with("sysLegalHold") || name.starts_with("sysDlpReview");
|
||||
assert!(
|
||||
!(name.ends_with("Update") && !holds)
|
||||
&& !(name.ends_with("Create") && !holds)
|
||||
@@ -249,7 +270,11 @@ mod tests {
|
||||
assert!(officer.contains(&hold));
|
||||
assert!(!tenant.contains(&hold));
|
||||
}
|
||||
for both in [Permission::SysComplianceGet, Permission::SysAuditGet, Permission::SysAccountGet] {
|
||||
for both in [
|
||||
Permission::SysComplianceGet,
|
||||
Permission::SysAuditGet,
|
||||
Permission::SysAccountGet,
|
||||
] {
|
||||
assert!(officer.contains(&both) && tenant.contains(&both));
|
||||
}
|
||||
assert!(!officer.contains(&Permission::SysAuditSettingsUpdate));
|
||||
@@ -257,9 +282,15 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn records_are_per_place() {
|
||||
let ValueClass::Any(server) = created_key(None) else { panic!() };
|
||||
let ValueClass::Any(a) = created_key(Some(Id::from(1u64))) else { panic!() };
|
||||
let ValueClass::Any(b) = created_key(Some(Id::from(2u64))) else { panic!() };
|
||||
let ValueClass::Any(server) = created_key(None) else {
|
||||
panic!()
|
||||
};
|
||||
let ValueClass::Any(a) = created_key(Some(Id::from(1u64))) else {
|
||||
panic!()
|
||||
};
|
||||
let ValueClass::Any(b) = created_key(Some(Id::from(2u64))) else {
|
||||
panic!()
|
||||
};
|
||||
assert_eq!(server.key, b"Pc");
|
||||
assert_ne!(a.key, b.key);
|
||||
assert!(a.key.starts_with(b"Pc"));
|
||||
|
||||
@@ -31,7 +31,8 @@ use types::id::Id;
|
||||
/// Granted to the default administrator roles: "Explain this"
|
||||
/// (ai-explain spec, EX-4: superuser by default), the audit log, account
|
||||
/// locks and legal holds (audit-hold-lock spec, AU-9, AL-12, LH-13), and
|
||||
/// the data inventory (personal-data catalog spec).
|
||||
/// the data inventory (personal-data catalog spec), and accepting security
|
||||
/// to-do items (security to-do list spec).
|
||||
const ADMIN_GRANTS: &[Permission] = &[
|
||||
Permission::SysAiExplain,
|
||||
Permission::SysAuditGet,
|
||||
@@ -46,6 +47,28 @@ const ADMIN_GRANTS: &[Permission] = &[
|
||||
Permission::SysLegalHoldUpdate,
|
||||
Permission::SysLegalHoldExport,
|
||||
Permission::SysComplianceGet,
|
||||
Permission::SysMailRuleGet,
|
||||
Permission::SysMailRuleUpdate,
|
||||
Permission::SysDlpPolicyGet,
|
||||
Permission::SysDlpPolicyUpdate,
|
||||
Permission::SysDlpReviewGet,
|
||||
Permission::SysDlpReviewUpdate,
|
||||
Permission::SysJournalGet,
|
||||
Permission::SysJournalUpdate,
|
||||
Permission::SysSecurityAccept,
|
||||
];
|
||||
|
||||
/// Granted to the server-level Compliance Officer role once it exists:
|
||||
/// seeing DLP rules and reviewing held mail (dlp-and-mail-flow-rules spec,
|
||||
/// §2.8, settled answer 4). A new install's role has them from the start.
|
||||
const OFFICER_GRANTS: &[Permission] = &[
|
||||
Permission::SysDlpPolicyGet,
|
||||
Permission::SysDlpReviewGet,
|
||||
Permission::SysDlpReviewUpdate,
|
||||
// journaling spec, JR-18: see journals, search and export them
|
||||
Permission::SysJournalGet,
|
||||
Permission::SysJournalSearch,
|
||||
Permission::SysJournalExport,
|
||||
];
|
||||
|
||||
/// Granted to the default tenant administrator roles: reading and exporting
|
||||
@@ -65,13 +88,16 @@ const TENANT_GRANTS: &[Permission] = &[
|
||||
enum Audience {
|
||||
Admin,
|
||||
Tenant,
|
||||
Officer,
|
||||
}
|
||||
|
||||
fn granted_key(permission: Permission, audience: Audience) -> ValueClass {
|
||||
let mut key = b"Pg".to_vec();
|
||||
// Admin grants keep the key they were first recorded under
|
||||
if audience == Audience::Tenant {
|
||||
key.extend_from_slice(b"tenant:");
|
||||
match audience {
|
||||
Audience::Admin => {}
|
||||
Audience::Tenant => key.extend_from_slice(b"tenant:"),
|
||||
Audience::Officer => key.extend_from_slice(b"officer:"),
|
||||
}
|
||||
key.extend_from_slice(permission.as_str().as_bytes());
|
||||
ValueClass::Any(AnyClass {
|
||||
@@ -82,7 +108,8 @@ fn granted_key(permission: Permission, audience: Audience) -> ValueClass {
|
||||
|
||||
pub(crate) async fn grant_new_admin_permissions(bp: &mut Bootstrap) -> trc::Result<()> {
|
||||
grant(bp, Audience::Admin, ADMIN_GRANTS).await?;
|
||||
grant(bp, Audience::Tenant, TENANT_GRANTS).await
|
||||
grant(bp, Audience::Tenant, TENANT_GRANTS).await?;
|
||||
grant(bp, Audience::Officer, OFFICER_GRANTS).await
|
||||
}
|
||||
|
||||
async fn grant(bp: &mut Bootstrap, audience: Audience, grants: &[Permission]) -> trc::Result<()> {
|
||||
@@ -101,10 +128,17 @@ async fn grant(bp: &mut Bootstrap, audience: Audience, grants: &[Permission]) ->
|
||||
if pending.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
// The officer role is the one the server made, if it has made it yet: a
|
||||
// new install makes it after this, with the permissions already in it
|
||||
let admin_roles: Vec<Id> = if audience == Audience::Officer {
|
||||
super::compliance_roles::server_role(&bp.data_store)
|
||||
.await?
|
||||
.into_iter()
|
||||
.collect()
|
||||
} else {
|
||||
// An administrator's default roles include the plain User role, which
|
||||
// every user also holds; only roles that are the audience's alone get it
|
||||
let admin_roles: Vec<Id> = bp
|
||||
.registry
|
||||
bp.registry
|
||||
.object::<Authentication>(Id::singleton())
|
||||
.await?
|
||||
.map(|auth| {
|
||||
@@ -118,7 +152,7 @@ async fn grant(bp: &mut Bootstrap, audience: Audience, grants: &[Permission]) ->
|
||||
]
|
||||
.concat(),
|
||||
),
|
||||
Audience::Tenant => (
|
||||
Audience::Tenant | Audience::Officer => (
|
||||
auth.default_tenant_role_ids.as_slice(),
|
||||
[
|
||||
auth.default_user_role_ids.as_slice(),
|
||||
@@ -133,7 +167,8 @@ async fn grant(bp: &mut Bootstrap, audience: Audience, grants: &[Permission]) ->
|
||||
.copied()
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
.unwrap_or_default()
|
||||
};
|
||||
// Fetched by id: the registry's listing doesn't reach stored roles
|
||||
for role_id in admin_roles {
|
||||
let Some(stored) = bp
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use crate::{
|
||||
@@ -72,11 +75,22 @@ impl Server {
|
||||
.acme_certificate_renewal_due(&domains, renew_before, now())
|
||||
.await?
|
||||
{
|
||||
return Err(AcmeError::NotDue(format!(
|
||||
"Certificate for domain {} is still valid; renewal is not due until {}",
|
||||
domain.name,
|
||||
UTCDateTime::from_timestamp(renew_at as i64)
|
||||
)));
|
||||
// INBUXA: a certificate already covering these names (one stored by
|
||||
// hand before the domain was switched to automatic, say) isn't a
|
||||
// failure: schedule the renewal for when it falls due. Returning
|
||||
// NotDue here ended the task for good, and nothing renewed the
|
||||
// certificate before it expired.
|
||||
trc::event!(
|
||||
Acme(trc::AcmeEvent::RenewBackoff),
|
||||
Domain = domain.name.clone(),
|
||||
Hostname = domains.as_slice(),
|
||||
Details = "A valid certificate already covers these names",
|
||||
NextRetry = trc::Value::Timestamp(renew_at),
|
||||
);
|
||||
return Ok(vec![Task::AcmeRenewal(TaskDomainManagement {
|
||||
domain_id,
|
||||
status: TaskStatus::at(renew_at as i64),
|
||||
})]);
|
||||
}
|
||||
|
||||
let dns_parameters = match &domain.dns_management {
|
||||
|
||||
@@ -0,0 +1,293 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! Whether the outside world can reach each node's ports (settings-reorg,
|
||||
//! Ports: the reachability check).
|
||||
//!
|
||||
//! A server can't answer this about itself: a connection to its own public
|
||||
//! address never leaves the machine, so it passes whatever the firewall in
|
||||
//! front says. In a cluster the other nodes are outside that machine. Every
|
||||
//! ten minutes each node resolves every other active node's hostname, as a
|
||||
//! sender would, and tries a TCP connection to each listener port on each
|
||||
//! address. What it saw goes in the shared in-memory store for an hour, under
|
||||
//! (target, prober), so whichever node the admin asks can report it all.
|
||||
//!
|
||||
//! A single server has no one outside to ask. It reports only whether each
|
||||
//! port is listening, and says so.
|
||||
//!
|
||||
//! A connection is all that's tried: nothing is sent, so no protocol logs a
|
||||
//! session and no rate limit counts it.
|
||||
|
||||
use crate::{KV_PORT_REACHABILITY, Server};
|
||||
use registry::schema::{enums::ClusterNodeStatus, structs::NetworkListener};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Value, json};
|
||||
use std::{
|
||||
collections::BTreeSet,
|
||||
net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use store::{dispatch::lookup::KeyValue, write::now};
|
||||
|
||||
/// How often each node probes the others.
|
||||
pub const PROBE_INTERVAL: Duration = Duration::from_secs(600);
|
||||
/// How long one node's view of another is kept: long enough to span a missed round.
|
||||
const KEEP_FOR: u64 = 3600;
|
||||
const CONNECT_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct Probe {
|
||||
pub port: u16,
|
||||
pub address: String,
|
||||
pub ok: bool,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct Report {
|
||||
/// Unix seconds.
|
||||
pub checked_at: u64,
|
||||
pub probes: Vec<Probe>,
|
||||
/// The hostname didn't resolve, so nothing could be tried.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// The ports a sender or client could reach: every listener's port, leaving
|
||||
/// out listeners bound only to loopback, which are private by design.
|
||||
pub fn public_ports<'x>(listeners: impl IntoIterator<Item = &'x NetworkListener>) -> Vec<u16> {
|
||||
listeners
|
||||
.into_iter()
|
||||
.flat_map(|l| l.bind.iter())
|
||||
.map(|addr| addr.0)
|
||||
.filter(|addr| !addr.ip().is_loopback())
|
||||
.map(|addr| addr.port())
|
||||
.collect::<BTreeSet<_>>()
|
||||
.into_iter()
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn key(target: &str, prober: &str) -> Vec<u8> {
|
||||
format!("{target}\n{prober}").into_bytes()
|
||||
}
|
||||
|
||||
async fn connect(address: SocketAddr) -> Result<(), String> {
|
||||
match tokio::time::timeout(CONNECT_TIMEOUT, tokio::net::TcpStream::connect(address)).await {
|
||||
Ok(Ok(_)) => Ok(()),
|
||||
Ok(Err(err)) => Err(err.to_string()),
|
||||
Err(_) => Err("no answer within 5 seconds".into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Tries each port on each address `hostname` resolves to.
|
||||
pub async fn probe_host(hostname: &str, ports: &[u16]) -> Report {
|
||||
let checked_at = now();
|
||||
let addresses = match tokio::net::lookup_host((hostname, 0)).await {
|
||||
Ok(found) => found.map(|a| a.ip()).collect::<BTreeSet<_>>(),
|
||||
Err(err) => {
|
||||
return Report {
|
||||
checked_at,
|
||||
probes: vec![],
|
||||
error: Some(format!("{hostname} doesn't resolve: {err}")),
|
||||
};
|
||||
}
|
||||
};
|
||||
let tries = addresses.iter().flat_map(|ip| {
|
||||
ports.iter().map(move |port| {
|
||||
let address = SocketAddr::new(*ip, *port);
|
||||
async move {
|
||||
let result = connect(address).await;
|
||||
Probe {
|
||||
port: *port,
|
||||
address: ip.to_string(),
|
||||
ok: result.is_ok(),
|
||||
error: result.err(),
|
||||
}
|
||||
}
|
||||
})
|
||||
});
|
||||
Report {
|
||||
checked_at,
|
||||
probes: futures::future::join_all(tries).await,
|
||||
error: None,
|
||||
}
|
||||
}
|
||||
|
||||
async fn listeners(server: &Server) -> trc::Result<Vec<NetworkListener>> {
|
||||
Ok(server
|
||||
.registry()
|
||||
.list::<NetworkListener>()
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|l| l.object)
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Where to knock to see a port listening on this machine: the bound
|
||||
/// address, or loopback of the same family for a wildcard bind.
|
||||
pub fn local_targets<'x>(
|
||||
listeners: impl IntoIterator<Item = &'x NetworkListener>,
|
||||
) -> Vec<SocketAddr> {
|
||||
listeners
|
||||
.into_iter()
|
||||
.flat_map(|l| l.bind.iter())
|
||||
.map(|addr| addr.0)
|
||||
.filter(|addr| !addr.ip().is_loopback())
|
||||
.map(|addr| match addr.ip() {
|
||||
IpAddr::V4(ip) if ip.is_unspecified() => {
|
||||
SocketAddr::new(Ipv4Addr::LOCALHOST.into(), addr.port())
|
||||
}
|
||||
IpAddr::V6(ip) if ip.is_unspecified() => {
|
||||
SocketAddr::new(Ipv6Addr::LOCALHOST.into(), addr.port())
|
||||
}
|
||||
_ => addr,
|
||||
})
|
||||
.collect::<BTreeSet<_>>()
|
||||
.into_iter()
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// One round: this node probes every other active node and records what it saw.
|
||||
pub async fn probe_peers(server: &Server) -> trc::Result<()> {
|
||||
let nodes = server.registry().cluster_node_list().await?;
|
||||
let me = server.registry().node_id() as u64;
|
||||
let Some(prober) = nodes
|
||||
.iter()
|
||||
.find(|n| n.node_id == me)
|
||||
.map(|n| n.hostname.clone())
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
let ports = public_ports(&listeners(server).await?);
|
||||
for target in nodes.iter().filter(|n| {
|
||||
n.node_id != me && n.status == ClusterNodeStatus::Active && n.hostname != prober
|
||||
}) {
|
||||
let report = probe_host(&target.hostname, &ports).await;
|
||||
server
|
||||
.in_memory_store()
|
||||
.key_set(
|
||||
KeyValue::with_prefix(
|
||||
KV_PORT_REACHABILITY,
|
||||
key(&target.hostname, &prober),
|
||||
serde_json::to_vec(&report).unwrap_or_default(),
|
||||
)
|
||||
.expires(KEEP_FOR),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// What `GET /api/ports/check` answers.
|
||||
pub async fn report(server: &Server) -> trc::Result<Value> {
|
||||
let listeners = listeners(server).await?;
|
||||
let ports = public_ports(&listeners);
|
||||
let nodes = if server.core.storage.coordinator.is_enabled() {
|
||||
server.registry().cluster_node_list().await?
|
||||
} else {
|
||||
vec![]
|
||||
};
|
||||
let active = nodes
|
||||
.iter()
|
||||
.filter(|n| n.status == ClusterNodeStatus::Active)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
if active.len() < 2 {
|
||||
// No one outside to ask: only whether each port is listening here.
|
||||
let started = Instant::now();
|
||||
let listening = futures::future::join_all(local_targets(&listeners).into_iter().map(
|
||||
|address| async move {
|
||||
let result = connect(address).await;
|
||||
json!({ "port": address.port(), "address": address.ip().to_string(), "listening": result.is_ok() })
|
||||
},
|
||||
))
|
||||
.await;
|
||||
return Ok(json!({
|
||||
"mode": "local",
|
||||
"ports": ports,
|
||||
"listening": listening,
|
||||
"ms": started.elapsed().as_millis() as u64,
|
||||
}));
|
||||
}
|
||||
|
||||
let mut out = Vec::new();
|
||||
for target in &active {
|
||||
let mut seen_by = Vec::new();
|
||||
for prober in active.iter().filter(|p| p.node_id != target.node_id) {
|
||||
let stored = server
|
||||
.in_memory_store()
|
||||
.key_get::<String>(KeyValue::<()>::build_key(
|
||||
KV_PORT_REACHABILITY,
|
||||
key(&target.hostname, &prober.hostname),
|
||||
))
|
||||
.await?;
|
||||
let report = stored.and_then(|raw| serde_json::from_str::<Report>(&raw).ok());
|
||||
seen_by.push(json!({ "prober": prober.hostname, "report": report }));
|
||||
}
|
||||
out.push(json!({ "hostname": target.hostname, "seenBy": seen_by }));
|
||||
}
|
||||
Ok(json!({
|
||||
"mode": "cluster",
|
||||
"ports": ports,
|
||||
"intervalSeconds": PROBE_INTERVAL.as_secs(),
|
||||
"nodes": out,
|
||||
}))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn listener(binds: &[&str]) -> NetworkListener {
|
||||
NetworkListener {
|
||||
bind: registry::schema::prelude::Map::new(
|
||||
binds.iter().map(|b| b.parse().unwrap()).collect(),
|
||||
),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn public_ports_leave_out_loopback_only_listeners() {
|
||||
let listeners = [
|
||||
listener(&["[::]:25"]),
|
||||
listener(&["0.0.0.0:993", "[::]:993"]),
|
||||
listener(&["127.0.0.1:8080"]),
|
||||
listener(&["203.0.113.5:465"]),
|
||||
];
|
||||
assert_eq!(public_ports(listeners.iter()), vec![25, 465, 993]);
|
||||
assert_eq!(
|
||||
local_targets(listeners.iter())
|
||||
.iter()
|
||||
.map(ToString::to_string)
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["127.0.0.1:993", "203.0.113.5:465", "[::1]:25", "[::1]:993"]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn probe_host_reports_open_and_closed_ports() {
|
||||
let open = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let open_port = open.local_addr().unwrap().port();
|
||||
let closed_port = {
|
||||
let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
l.local_addr().unwrap().port()
|
||||
};
|
||||
let report = probe_host("127.0.0.1", &[open_port, closed_port]).await;
|
||||
assert_eq!(report.error, None);
|
||||
let ok = |port| report.probes.iter().find(|p| p.port == port).unwrap().ok;
|
||||
assert!(ok(open_port));
|
||||
assert!(!ok(closed_port));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn probe_host_says_when_a_name_does_not_resolve() {
|
||||
let report = probe_host("does-not-exist.invalid", &[25]).await;
|
||||
assert!(report.probes.is_empty());
|
||||
assert!(report.error.unwrap().contains("doesn't resolve"));
|
||||
}
|
||||
}
|
||||
@@ -156,15 +156,7 @@ async fn post_webhook_events(
|
||||
|
||||
// Add HMAC-SHA256 signature
|
||||
let mut headers = settings.headers.clone();
|
||||
if !settings.key.is_empty() {
|
||||
let key = hmac::Key::new(hmac::HMAC_SHA256, settings.key.as_bytes());
|
||||
let tag = hmac::sign(&key, body.as_bytes());
|
||||
|
||||
headers.insert(
|
||||
"X-Signature",
|
||||
STANDARD.encode(tag.as_ref()).parse().unwrap(),
|
||||
);
|
||||
}
|
||||
sign(&mut headers, &settings.key, &body);
|
||||
|
||||
// Send request
|
||||
let response = settings
|
||||
@@ -188,3 +180,150 @@ async fn post_webhook_events(
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Adds the HMAC-SHA256 `X-Signature` a receiver checks, when the webhook has a key.
|
||||
fn sign(headers: &mut hyper::HeaderMap, key: &str, body: &str) {
|
||||
if !key.is_empty() {
|
||||
let key = hmac::Key::new(hmac::HMAC_SHA256, key.as_bytes());
|
||||
let tag = hmac::sign(&key, body.as_bytes());
|
||||
|
||||
headers.insert(
|
||||
"X-Signature",
|
||||
STANDARD.encode(tag.as_ref()).parse().unwrap(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// inbuxa: "Send test" for a saved webhook (settings-reorg, Webhooks). One
|
||||
/// sample event, sent the way a real batch is: the same URL, headers, sign-in,
|
||||
/// signature, timeout and certificate checks. The event's type,
|
||||
/// `webhook.test`, is none the server raises, and an `X-Inbuxa-Test` header
|
||||
/// marks it, so a receiver can tell it apart. Answers the HTTP status, or why
|
||||
/// nothing came back.
|
||||
pub async fn send_test(hook: ®istry::schema::structs::WebHook) -> Result<u16, String> {
|
||||
let mut headers = hook
|
||||
.http_auth
|
||||
.build_headers(hook.http_headers.clone(), "application/json".into())
|
||||
.await
|
||||
.map_err(|err| format!("Unable to build HTTP headers: {err}"))?;
|
||||
let key = hook
|
||||
.signature_key
|
||||
.secret()
|
||||
.await
|
||||
.map_err(|err| format!("Unable to retrieve signature key: {err}"))?
|
||||
.unwrap_or_default()
|
||||
.into_owned();
|
||||
|
||||
let created = now();
|
||||
let body = serde_json::json!({
|
||||
"events": [{
|
||||
"id": format!("test-{created}"),
|
||||
"createdAt": mail_parser::DateTime::from_timestamp(created as i64).to_rfc3339(),
|
||||
"type": "webhook.test",
|
||||
"data": { "details": "A test from inbuxa Admin. Nothing happened on the server." },
|
||||
}]
|
||||
})
|
||||
.to_string();
|
||||
sign(&mut headers, &key, &body);
|
||||
headers.insert("X-Inbuxa-Test", "true".parse().unwrap());
|
||||
|
||||
let response = utils::http::http_client_builder(hook.allow_invalid_certs)
|
||||
.build()
|
||||
.map_err(|err| format!("Unable to build an HTTP client: {err}"))?
|
||||
.post(&hook.url)
|
||||
.timeout(hook.timeout.into_inner())
|
||||
.headers(headers)
|
||||
.body(body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| format!("Webhook request to {} failed: {err}", hook.url))?;
|
||||
Ok(response.status().as_u16())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use registry::schema::structs::{SecretKeyOptional, SecretKeyValue, WebHook};
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
|
||||
/// One request in, the given status out; hands back what was received.
|
||||
async fn receiver(status: &'static str) -> (String, tokio::task::JoinHandle<String>) {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!("http://{}/hook", listener.local_addr().unwrap());
|
||||
let task = tokio::spawn(async move {
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
let mut buf = Vec::new();
|
||||
let mut chunk = [0u8; 4096];
|
||||
loop {
|
||||
let n = socket.read(&mut chunk).await.unwrap();
|
||||
buf.extend_from_slice(&chunk[..n]);
|
||||
let text = String::from_utf8_lossy(&buf);
|
||||
if let Some(end) = text.find("\r\n\r\n") {
|
||||
let length = text[..end]
|
||||
.lines()
|
||||
.find_map(|l| {
|
||||
l.to_ascii_lowercase()
|
||||
.strip_prefix("content-length:")
|
||||
.map(|v| v.trim().parse::<usize>().unwrap())
|
||||
})
|
||||
.unwrap_or(0);
|
||||
if buf.len() >= end + 4 + length || n == 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
socket
|
||||
.write_all(
|
||||
format!("HTTP/1.1 {status}\r\ncontent-length: 0\r\nconnection: close\r\n\r\n")
|
||||
.as_bytes(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
String::from_utf8_lossy(&buf).into_owned()
|
||||
});
|
||||
(url, task)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn send_test_signs_and_marks_the_sample() {
|
||||
let (url, task) = receiver("204 No Content").await;
|
||||
let hook = WebHook {
|
||||
url,
|
||||
enable: false,
|
||||
signature_key: SecretKeyOptional::Value(SecretKeyValue { secret: "k".into() }),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(send_test(&hook).await, Ok(204));
|
||||
|
||||
let request = task.await.unwrap();
|
||||
let (head, body) = request.split_once("\r\n\r\n").unwrap();
|
||||
let head = head.to_ascii_lowercase();
|
||||
assert!(head.contains("x-inbuxa-test: true"), "{head}");
|
||||
let parsed: serde_json::Value = serde_json::from_str(body).unwrap();
|
||||
assert_eq!(parsed["events"][0]["type"], "webhook.test");
|
||||
let tag = hmac::sign(&hmac::Key::new(hmac::HMAC_SHA256, b"k"), body.as_bytes());
|
||||
assert!(
|
||||
head.contains(&format!(
|
||||
"x-signature: {}",
|
||||
STANDARD.encode(tag.as_ref()).to_ascii_lowercase()
|
||||
)),
|
||||
"{head}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn send_test_reports_what_came_back() {
|
||||
let (url, _task) = receiver("403 Forbidden").await;
|
||||
let hook = WebHook {
|
||||
url,
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(send_test(&hook).await, Ok(403));
|
||||
|
||||
let hook = WebHook {
|
||||
url: "http://127.0.0.1:9/hook".into(),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(send_test(&hook).await.unwrap_err().contains("failed"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,6 +21,13 @@ base64 = "0.23"
|
||||
sha2 = "0.11"
|
||||
flate2 = "1.1"
|
||||
tokio = { version = "1.53", features = ["sync", "rt"] }
|
||||
# inbuxa: DLP detectors and attachment text (dlp-and-mail-flow-rules spec)
|
||||
regex = "1.13.1"
|
||||
aho-corasick = "1.1"
|
||||
zip = "8.6"
|
||||
quick-xml = "0.41"
|
||||
mail-parser = { version = "0.11", features = ["full_encoding"] }
|
||||
mail-builder = { version = "1.0" }
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { version = "1.53", features = ["macros", "rt"] }
|
||||
|
||||
@@ -0,0 +1,689 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! The built-in journal (JR-5, JR-6, JR-13). Keys, after `J`:
|
||||
//!
|
||||
//! - `e` + node + seq: a chain link: its seq, the hash of the link before
|
||||
//! it, and the SHA-256 of its entry. One chain per node, as the audit log
|
||||
//! keeps (AU-6), but a link names its entry by hash instead of holding it,
|
||||
//! so an entry can go at the end of its own retention without breaking
|
||||
//! the chain: entries don't expire in chain order.
|
||||
//! - `c` + node + seq: the entry, as JSON; its bytes are what the link's
|
||||
//! hash names.
|
||||
//! - `p` + node + seq: when an entry past its retention was purged. A link
|
||||
//! whose entry is gone without this marker is a broken chain.
|
||||
//! - `t` + time + node + seq: the time index, for search.
|
||||
//! - `x` + expiry + node + seq: the expiry index, for purge.
|
||||
//! - `h` + node: the chain's head: its hash, then its seq as the last eight
|
||||
//! bytes, which each append asserts.
|
||||
//! - `f` + node: where the chain starts after purged links at its start
|
||||
//! were cleared, and the hash the first kept link names.
|
||||
//!
|
||||
//! The report itself is a blob, kept by a temporary link that lasts until
|
||||
//! its entry is purged. Nothing here changes or removes an entry before
|
||||
//! its time; nothing in JMAP can.
|
||||
|
||||
use super::{Direction, FEATURE, Json};
|
||||
use crate::hold::HELD_UNTIL;
|
||||
use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::fmt;
|
||||
use store::{
|
||||
BlobStore, Deserialize, IterateParams, SUBSPACE_INBUXA, Serialize, Store, ValueKey,
|
||||
write::{AnyClass, BatchBuilder, BlobLink, BlobOp, ValueClass, assert::AssertValue},
|
||||
};
|
||||
use tokio::sync::Mutex;
|
||||
use trc::AddContext;
|
||||
use types::blob_hash::BlobHash;
|
||||
|
||||
const KIND_LINK: u8 = b'e';
|
||||
const KIND_CONTENT: u8 = b'c';
|
||||
const KIND_PURGED: u8 = b'p';
|
||||
const KIND_TIME: u8 = b't';
|
||||
const KIND_EXPIRY: u8 = b'x';
|
||||
const KIND_HEAD: u8 = b'h';
|
||||
const KIND_FLOOR: u8 = b'f';
|
||||
|
||||
const APPEND_ATTEMPTS: usize = 5;
|
||||
/// Entries purged per batch.
|
||||
const PURGE_BATCH: usize = 100;
|
||||
|
||||
/// Where one entry sits: its node's chain and its place in it.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
|
||||
pub struct EntryId {
|
||||
pub node: u64,
|
||||
pub seq: u64,
|
||||
}
|
||||
|
||||
impl EntryId {
|
||||
/// As one number, for JMAP ids: the node in the top 16 bits.
|
||||
pub fn to_u64(&self) -> u64 {
|
||||
(self.node << 48) | (self.seq & ((1 << 48) - 1))
|
||||
}
|
||||
|
||||
pub fn from_u64(id: u64) -> Self {
|
||||
EntryId {
|
||||
node: id >> 48,
|
||||
seq: id & ((1 << 48) - 1),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for EntryId {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{}-{}", self.node, self.seq)
|
||||
}
|
||||
}
|
||||
|
||||
/// One journaled message (JR-5).
|
||||
#[derive(Debug, Clone, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Entry {
|
||||
pub queue_id: u64,
|
||||
/// Seconds.
|
||||
pub at: u64,
|
||||
pub direction: Direction,
|
||||
pub sender: String,
|
||||
pub authenticated: bool,
|
||||
pub recipients: Vec<String>,
|
||||
pub subject: String,
|
||||
pub message_id: String,
|
||||
/// The people here on either side, whose holds keep the entry.
|
||||
pub accounts: Vec<u32>,
|
||||
pub tenants: Vec<u32>,
|
||||
/// The journals that took it.
|
||||
pub journals: Vec<u32>,
|
||||
pub held: bool,
|
||||
/// The report's blob, hex.
|
||||
pub blob: String,
|
||||
pub size: u64,
|
||||
/// SHA-256 of the report, hex.
|
||||
pub sha256: String,
|
||||
/// Seconds.
|
||||
pub expires_at: u64,
|
||||
}
|
||||
|
||||
impl Entry {
|
||||
pub fn blob_hash(&self) -> Option<BlobHash> {
|
||||
let bytes = unhex(&self.blob)?;
|
||||
BlobHash::try_from_hash_slice(&bytes).ok()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct Link {
|
||||
seq: u64,
|
||||
prev: String,
|
||||
content: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, SerdeSerialize, SerdeDeserialize)]
|
||||
struct Floor {
|
||||
seq: u64,
|
||||
prev: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq)]
|
||||
struct Head {
|
||||
seq: u64,
|
||||
hash: String,
|
||||
}
|
||||
|
||||
impl Head {
|
||||
fn to_bytes(&self) -> Vec<u8> {
|
||||
let mut bytes = self.hash.as_bytes().to_vec();
|
||||
bytes.extend_from_slice(&self.seq.to_be_bytes());
|
||||
bytes
|
||||
}
|
||||
}
|
||||
|
||||
impl Deserialize for Head {
|
||||
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
|
||||
let split = bytes.len().checked_sub(8).ok_or_else(|| {
|
||||
trc::StoreEvent::DataCorruption
|
||||
.into_err()
|
||||
.details("Invalid journal chain head")
|
||||
})?;
|
||||
Ok(Head {
|
||||
seq: u64::from_be_bytes(bytes[split..].try_into().unwrap()),
|
||||
hash: String::from_utf8_lossy(&bytes[..split]).into_owned(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct Raw(Vec<u8>);
|
||||
|
||||
impl Deserialize for Raw {
|
||||
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
|
||||
Ok(Raw(bytes.to_vec()))
|
||||
}
|
||||
}
|
||||
|
||||
fn class(kind: u8, parts: &[u64]) -> ValueClass {
|
||||
let mut key = Vec::with_capacity(2 + parts.len() * 8);
|
||||
key.push(FEATURE);
|
||||
key.push(kind);
|
||||
for part in parts {
|
||||
key.extend_from_slice(&part.to_be_bytes());
|
||||
}
|
||||
ValueClass::Any(AnyClass {
|
||||
subspace: SUBSPACE_INBUXA,
|
||||
key,
|
||||
})
|
||||
}
|
||||
|
||||
fn key(kind: u8, parts: &[u64]) -> ValueKey<ValueClass> {
|
||||
ValueKey::from(class(kind, parts))
|
||||
}
|
||||
|
||||
/// Where an entry's content is kept, for tests that check tampering shows.
|
||||
pub fn content_key(id: EntryId) -> ValueKey<ValueClass> {
|
||||
key(KIND_CONTENT, &[id.node, id.seq])
|
||||
}
|
||||
|
||||
/// The numbers after the kind byte, from the key's tail.
|
||||
fn parse_key(key: &[u8], kind: u8, parts: usize) -> Option<Vec<u64>> {
|
||||
let len = 2 + parts * 8;
|
||||
let tail = key.get(key.len().checked_sub(len)?..)?;
|
||||
(tail[0] == FEATURE && tail[1] == kind).then_some(())?;
|
||||
Some(
|
||||
tail[2..]
|
||||
.chunks_exact(8)
|
||||
.map(|chunk| u64::from_be_bytes(chunk.try_into().unwrap()))
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn hex(bytes: &[u8]) -> String {
|
||||
bytes.iter().map(|b| format!("{b:02x}")).collect()
|
||||
}
|
||||
|
||||
fn unhex(value: &str) -> Option<Vec<u8>> {
|
||||
(value.len() % 2 == 0).then_some(())?;
|
||||
(0..value.len())
|
||||
.step_by(2)
|
||||
.map(|i| u8::from_str_radix(value.get(i..i + 2)?, 16).ok())
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn sha256(bytes: &[u8]) -> String {
|
||||
hex(&Sha256::digest(bytes))
|
||||
}
|
||||
|
||||
async fn head(data: &Store, node: u64) -> trc::Result<Option<Head>> {
|
||||
data.get_value::<Head>(key(KIND_HEAD, &[node]))
|
||||
.await
|
||||
.caused_by(trc::location!())
|
||||
}
|
||||
|
||||
async fn floor(data: &Store, node: u64) -> trc::Result<Floor> {
|
||||
Ok(data
|
||||
.get_value::<Json<Floor>>(key(KIND_FLOOR, &[node]))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.map(|Json(floor)| floor)
|
||||
.unwrap_or(Floor {
|
||||
seq: 1,
|
||||
prev: String::new(),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn nodes(data: &Store) -> trc::Result<Vec<u64>> {
|
||||
let mut nodes = Vec::new();
|
||||
data.iterate(
|
||||
IterateParams::new(key(KIND_HEAD, &[0]), key(KIND_HEAD, &[u64::MAX])).no_values(),
|
||||
|key, _| {
|
||||
if let Some(parts) = parse_key(key, KIND_HEAD, 1) {
|
||||
nodes.push(parts[0]);
|
||||
}
|
||||
Ok(true)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
Ok(nodes)
|
||||
}
|
||||
|
||||
/// Lines up this process's appends; the store's assert settles the rest.
|
||||
static APPENDING: Mutex<()> = Mutex::const_new(());
|
||||
|
||||
/// Adds an entry to this node's chain, and links its report's blob (already
|
||||
/// written) until the entry is purged. An error means nothing was written.
|
||||
pub async fn append(data: &Store, node: u64, entry: &Entry) -> trc::Result<EntryId> {
|
||||
let blob = entry.blob_hash().ok_or_else(|| {
|
||||
trc::StoreEvent::UnexpectedError
|
||||
.into_err()
|
||||
.details("Journal entry without a blob")
|
||||
})?;
|
||||
let content = Json(entry).serialize()?;
|
||||
let content_hash = sha256(&content);
|
||||
let _appending = APPENDING.lock().await;
|
||||
let mut attempt = 0;
|
||||
loop {
|
||||
attempt += 1;
|
||||
let current = head(data, node).await?;
|
||||
let (seq, prev) = current
|
||||
.as_ref()
|
||||
.map_or((1, String::new()), |head| (head.seq + 1, head.hash.clone()));
|
||||
let link = Json(&Link {
|
||||
seq,
|
||||
prev,
|
||||
content: content_hash.clone(),
|
||||
})
|
||||
.serialize()?;
|
||||
let new_head = Head {
|
||||
seq,
|
||||
hash: sha256(&link),
|
||||
};
|
||||
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.assert_value(
|
||||
class(KIND_HEAD, &[node]),
|
||||
current.map_or(AssertValue::None, |head| AssertValue::U64(head.seq)),
|
||||
);
|
||||
batch
|
||||
.set(class(KIND_LINK, &[node, seq]), link)
|
||||
.set(class(KIND_CONTENT, &[node, seq]), content.clone())
|
||||
.set(class(KIND_TIME, &[entry.at, node, seq]), vec![])
|
||||
.set(class(KIND_EXPIRY, &[entry.expires_at, node, seq]), vec![])
|
||||
.set(class(KIND_HEAD, &[node]), new_head.to_bytes())
|
||||
.set(
|
||||
BlobOp::Link {
|
||||
hash: blob.clone(),
|
||||
to: BlobLink::Temporary { until: HELD_UNTIL },
|
||||
},
|
||||
vec![],
|
||||
)
|
||||
.set(BlobOp::Commit { hash: blob.clone() }, vec![]);
|
||||
match data.write(batch.build_all()).await {
|
||||
Ok(_) => return Ok(EntryId { node, seq }),
|
||||
Err(err)
|
||||
if attempt < APPEND_ATTEMPTS
|
||||
&& matches!(
|
||||
err.as_ref(),
|
||||
trc::EventType::Store(trc::StoreEvent::AssertValueFailed)
|
||||
) => {}
|
||||
Err(err) => return Err(err.caused_by(trc::location!())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One entry, unless it was purged.
|
||||
pub async fn get(data: &Store, id: EntryId) -> trc::Result<Option<Entry>> {
|
||||
Ok(data
|
||||
.get_value::<Json<Entry>>(key(KIND_CONTENT, &[id.node, id.seq]))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.map(|Json(entry)| entry))
|
||||
}
|
||||
|
||||
/// Entries written in `[after, before)` (seconds), newest first, up to
|
||||
/// `limit`.
|
||||
pub async fn list(
|
||||
data: &Store,
|
||||
after: u64,
|
||||
before: u64,
|
||||
limit: usize,
|
||||
) -> trc::Result<Vec<(EntryId, Entry)>> {
|
||||
let mut ids = Vec::new();
|
||||
data.iterate(
|
||||
IterateParams::new(
|
||||
key(KIND_TIME, &[after, 0, 0]),
|
||||
key(KIND_TIME, &[before.saturating_sub(1), u64::MAX, u64::MAX]),
|
||||
)
|
||||
.descending()
|
||||
.no_values(),
|
||||
|key, _| {
|
||||
if let Some(parts) = parse_key(key, KIND_TIME, 3) {
|
||||
ids.push(EntryId {
|
||||
node: parts[1],
|
||||
seq: parts[2],
|
||||
});
|
||||
}
|
||||
Ok(ids.len() < limit)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
let mut out = Vec::with_capacity(ids.len());
|
||||
for id in ids {
|
||||
if let Some(entry) = get(data, id).await? {
|
||||
out.push((id, entry));
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// What a purge did.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct Purged {
|
||||
pub removed: usize,
|
||||
/// Past their time, kept for a legal hold.
|
||||
pub kept_for_hold: usize,
|
||||
}
|
||||
|
||||
/// Removes entries past their retention (JR-13), except those `held` keeps:
|
||||
/// the entry, its indexes and its blob's link go; the chain link stays,
|
||||
/// with a purge marker. Then each chain's start moves past purged links.
|
||||
pub async fn purge(
|
||||
data: &Store,
|
||||
now: u64,
|
||||
held: impl Fn(&Entry) -> bool + Sync + Send,
|
||||
) -> trc::Result<Purged> {
|
||||
let mut due = Vec::new();
|
||||
data.iterate(
|
||||
IterateParams::new(
|
||||
key(KIND_EXPIRY, &[0, 0, 0]),
|
||||
key(KIND_EXPIRY, &[now, u64::MAX, u64::MAX]),
|
||||
)
|
||||
.ascending()
|
||||
.no_values(),
|
||||
|key, _| {
|
||||
if let Some(parts) = parse_key(key, KIND_EXPIRY, 3) {
|
||||
due.push((
|
||||
parts[0],
|
||||
EntryId {
|
||||
node: parts[1],
|
||||
seq: parts[2],
|
||||
},
|
||||
));
|
||||
}
|
||||
Ok(due.len() < 100_000)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
let mut purged = Purged::default();
|
||||
for chunk in due.chunks(PURGE_BATCH) {
|
||||
let mut batch = BatchBuilder::new();
|
||||
for (expires_at, id) in chunk {
|
||||
let parts = [id.node, id.seq];
|
||||
let Some(entry) = get(data, *id).await? else {
|
||||
// Its entry is already gone: only the index is left
|
||||
batch.clear(class(KIND_EXPIRY, &[*expires_at, id.node, id.seq]));
|
||||
continue;
|
||||
};
|
||||
if held(&entry) {
|
||||
purged.kept_for_hold += 1;
|
||||
continue;
|
||||
}
|
||||
batch
|
||||
.clear(class(KIND_CONTENT, &parts))
|
||||
.clear(class(KIND_TIME, &[entry.at, id.node, id.seq]))
|
||||
.clear(class(KIND_EXPIRY, &[*expires_at, id.node, id.seq]))
|
||||
.set(class(KIND_PURGED, &parts), now.to_be_bytes().to_vec());
|
||||
if let Some(blob) = entry.blob_hash() {
|
||||
batch.clear(BlobOp::Link {
|
||||
hash: blob,
|
||||
to: BlobLink::Temporary { until: HELD_UNTIL },
|
||||
});
|
||||
}
|
||||
purged.removed += 1;
|
||||
}
|
||||
if !batch.is_empty() {
|
||||
data.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
}
|
||||
}
|
||||
|
||||
for node in nodes(data).await? {
|
||||
advance_floor(data, node).await?;
|
||||
}
|
||||
Ok(purged)
|
||||
}
|
||||
|
||||
/// Clears the purged links at the start of a node's chain, recording where
|
||||
/// it now starts and the hash that start names.
|
||||
async fn advance_floor(data: &Store, node: u64) -> trc::Result<()> {
|
||||
let start = floor(data, node).await?;
|
||||
let mut cleared: Vec<u64> = Vec::new();
|
||||
let mut next = start.clone();
|
||||
let mut purged_seqs = Vec::new();
|
||||
data.iterate(
|
||||
IterateParams::new(
|
||||
key(KIND_PURGED, &[node, start.seq]),
|
||||
key(KIND_PURGED, &[node, u64::MAX]),
|
||||
)
|
||||
.ascending()
|
||||
.no_values(),
|
||||
|key, _| {
|
||||
if let Some(parts) = parse_key(key, KIND_PURGED, 2) {
|
||||
purged_seqs.push(parts[1]);
|
||||
}
|
||||
Ok(purged_seqs.len() < 100_000)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
for seq in purged_seqs {
|
||||
if seq != next.seq {
|
||||
break;
|
||||
}
|
||||
let Some(Raw(link)) = data
|
||||
.get_value::<Raw>(key(KIND_LINK, &[node, seq]))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
else {
|
||||
break;
|
||||
};
|
||||
next = Floor {
|
||||
seq: seq + 1,
|
||||
prev: sha256(&link),
|
||||
};
|
||||
cleared.push(seq);
|
||||
}
|
||||
if cleared.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
// The floor moves first: a run cut short leaves links before it, which
|
||||
// the next run clears, never a chain that looks broken
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.set(class(KIND_FLOOR, &[node]), Json(&next).serialize()?);
|
||||
data.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
for chunk in cleared.chunks(PURGE_BATCH) {
|
||||
let mut batch = BatchBuilder::new();
|
||||
for seq in chunk {
|
||||
batch
|
||||
.clear(class(KIND_LINK, &[node, *seq]))
|
||||
.clear(class(KIND_PURGED, &[node, *seq]));
|
||||
}
|
||||
data.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// One node's chain, as [`verify`] found it.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, SerdeSerialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ChainReport {
|
||||
pub node: u64,
|
||||
pub entries: u64,
|
||||
pub purged: u64,
|
||||
pub first_seq: u64,
|
||||
pub last_seq: u64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub broken_at: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub reason: Option<String>,
|
||||
}
|
||||
|
||||
/// Rechecks every node's chain (JR-6): each link names the hash of the one
|
||||
/// before it, seqs run without gaps, the head matches the last link, each
|
||||
/// entry hashes to what its link names or was purged, and, with `blobs`,
|
||||
/// each report is there and hashes to what its entry names.
|
||||
pub async fn verify(data: &Store, blobs: Option<&BlobStore>) -> trc::Result<Vec<ChainReport>> {
|
||||
let mut reports = Vec::new();
|
||||
for node in nodes(data).await? {
|
||||
let start = floor(data, node).await?;
|
||||
let head = head(data, node).await?.unwrap_or_default();
|
||||
let mut report = ChainReport {
|
||||
node,
|
||||
entries: 0,
|
||||
purged: 0,
|
||||
first_seq: start.seq,
|
||||
last_seq: start.seq.saturating_sub(1),
|
||||
broken_at: None,
|
||||
reason: None,
|
||||
};
|
||||
let mut links = Vec::new();
|
||||
data.iterate(
|
||||
IterateParams::new(
|
||||
key(KIND_LINK, &[node, start.seq]),
|
||||
key(KIND_LINK, &[node, u64::MAX]),
|
||||
)
|
||||
.ascending(),
|
||||
|key, value| {
|
||||
if let Some(parts) = parse_key(key, KIND_LINK, 2) {
|
||||
links.push((parts[1], value.to_vec()));
|
||||
}
|
||||
Ok(true)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
let mut expected_seq = start.seq;
|
||||
let mut expected_prev = start.prev.clone();
|
||||
for (seq, bytes) in links {
|
||||
let broken = |report: &mut ChainReport, reason: &str| {
|
||||
report.broken_at = Some(EntryId { node, seq }.to_string());
|
||||
report.reason = Some(reason.to_string());
|
||||
};
|
||||
let Ok(Json(link)) = Json::<Link>::deserialize(&bytes) else {
|
||||
broken(&mut report, "The link can't be read.");
|
||||
break;
|
||||
};
|
||||
if seq != expected_seq || link.seq != seq {
|
||||
report.broken_at = Some(EntryId { node, seq }.to_string());
|
||||
report.reason = Some(format!(
|
||||
"Entry {expected_seq} is missing; the next one found is {seq}."
|
||||
));
|
||||
break;
|
||||
}
|
||||
if link.prev != expected_prev {
|
||||
broken(
|
||||
&mut report,
|
||||
"The link doesn't follow from the one before it: one of them was changed.",
|
||||
);
|
||||
break;
|
||||
}
|
||||
match data
|
||||
.get_value::<Raw>(key(KIND_CONTENT, &[node, seq]))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
{
|
||||
Some(Raw(content)) => {
|
||||
if sha256(&content) != link.content {
|
||||
broken(&mut report, "The entry was changed after it was written.");
|
||||
break;
|
||||
}
|
||||
if let Some(blobs) = blobs {
|
||||
let Ok(Json(entry)) = Json::<Entry>::deserialize(&content) else {
|
||||
broken(&mut report, "The entry can't be read.");
|
||||
break;
|
||||
};
|
||||
let report_bytes = match entry.blob_hash() {
|
||||
Some(hash) => blobs
|
||||
.get_blob(hash.as_slice(), 0..usize::MAX)
|
||||
.await
|
||||
.caused_by(trc::location!())?,
|
||||
None => None,
|
||||
};
|
||||
match report_bytes {
|
||||
Some(bytes) if sha256(&bytes) == entry.sha256 => {}
|
||||
Some(_) => {
|
||||
broken(&mut report, "The report doesn't match its entry.");
|
||||
break;
|
||||
}
|
||||
None => {
|
||||
broken(&mut report, "The report is missing.");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
report.entries += 1;
|
||||
}
|
||||
None => {
|
||||
if data
|
||||
.get_value::<Raw>(key(KIND_PURGED, &[node, seq]))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.is_none()
|
||||
{
|
||||
broken(&mut report, "The entry was removed before its time.");
|
||||
break;
|
||||
}
|
||||
report.purged += 1;
|
||||
}
|
||||
}
|
||||
expected_prev = sha256(&bytes);
|
||||
expected_seq = seq + 1;
|
||||
report.last_seq = seq;
|
||||
}
|
||||
|
||||
if report.broken_at.is_none()
|
||||
&& (head.seq != report.last_seq
|
||||
|| (report.last_seq >= report.first_seq && head.hash != expected_prev))
|
||||
{
|
||||
report.broken_at = Some(
|
||||
EntryId {
|
||||
node,
|
||||
seq: report.last_seq,
|
||||
}
|
||||
.to_string(),
|
||||
);
|
||||
report.reason = Some(
|
||||
"The chain's recorded end doesn't match its last link: entries were removed \
|
||||
or changed at the end."
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
reports.push(report);
|
||||
}
|
||||
Ok(reports)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn keys_read_back() {
|
||||
let ValueClass::Any(any) = class(KIND_EXPIRY, &[5, 3, 9]) else {
|
||||
panic!()
|
||||
};
|
||||
assert_eq!(parse_key(&any.key, KIND_EXPIRY, 3), Some(vec![5, 3, 9]));
|
||||
let mut with_subspace = vec![SUBSPACE_INBUXA];
|
||||
with_subspace.extend_from_slice(&any.key);
|
||||
assert_eq!(
|
||||
parse_key(&with_subspace, KIND_EXPIRY, 3),
|
||||
Some(vec![5, 3, 9])
|
||||
);
|
||||
assert_eq!(parse_key(&any.key, KIND_TIME, 3), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hex_round_trips() {
|
||||
let bytes = [0u8, 1, 0xab, 0xff];
|
||||
assert_eq!(unhex(&hex(&bytes)), Some(bytes.to_vec()));
|
||||
assert_eq!(unhex("abc"), None);
|
||||
assert_eq!(unhex("zz"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ids_read_back() {
|
||||
let id = EntryId { node: 3, seq: 77 };
|
||||
assert_eq!(EntryId::from_u64(id.to_u64()), id);
|
||||
assert_eq!(id.to_string(), "3-77");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,431 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! Journaling (journaling spec, JR-1 to JR-18): a copy of each message the
|
||||
//! server queues, with its envelope, kept where nothing in the product
|
||||
//! changes or removes it before its retention ends.
|
||||
//!
|
||||
//! - this module: journals, what makes one valid, and where they're kept;
|
||||
//! - [`report`]: the journal report around the untouched message (JR-3);
|
||||
//! - [`entries`]: the built-in journal and its chain (JR-5, JR-6, JR-13).
|
||||
//!
|
||||
//! Kept in the fork's subspace (`store::SUBSPACE_INBUXA`). Every key starts
|
||||
//! with `J`; journals are `j` + id (u32), as JSON. There are few, so they're
|
||||
//! read whole.
|
||||
|
||||
pub mod entries;
|
||||
pub mod report;
|
||||
|
||||
use crate::{hold::Member, mailflow::rules::jmap_ids};
|
||||
use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize, de::DeserializeOwned};
|
||||
use std::{
|
||||
sync::{Arc, RwLock},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use store::{
|
||||
Deserialize, IterateParams, SUBSPACE_INBUXA, Serialize, Store, ValueKey,
|
||||
write::{AnyClass, BatchBuilder, ValueClass, assert::AssertValue},
|
||||
};
|
||||
use trc::AddContext;
|
||||
|
||||
pub(crate) const FEATURE: u8 = b'J';
|
||||
const KIND_JOURNAL: u8 = b'j';
|
||||
const CREATE_ATTEMPTS: usize = 5;
|
||||
|
||||
/// Retention a journal may be given, in days (settled answer 3).
|
||||
pub const MIN_RETENTION_DAYS: u32 = 30;
|
||||
pub const MAX_RETENTION_DAYS: u32 = 3650;
|
||||
/// Most entries in one scope list.
|
||||
const MAX_LIST: usize = 5_000;
|
||||
|
||||
/// Which way a message goes, from this server's side (JR-9).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum Direction {
|
||||
/// From someone here to at least one recipient elsewhere.
|
||||
Outgoing,
|
||||
/// From elsewhere to someone here.
|
||||
Incoming,
|
||||
/// From someone here, to people here only.
|
||||
Internal,
|
||||
Any,
|
||||
}
|
||||
|
||||
impl Direction {
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Direction::Outgoing => "outgoing",
|
||||
Direction::Incoming => "incoming",
|
||||
Direction::Internal => "internal",
|
||||
Direction::Any => "any",
|
||||
}
|
||||
}
|
||||
|
||||
/// A message's direction: `Any` is never one.
|
||||
pub fn of(sender_local: bool, any_remote: bool, any_local: bool) -> Direction {
|
||||
match (sender_local, any_remote) {
|
||||
(true, true) => Direction::Outgoing,
|
||||
(true, false) => Direction::Internal,
|
||||
(false, _) if any_local => Direction::Incoming,
|
||||
// Nobody here on either side: relayed mail counts as outgoing
|
||||
(false, _) => Direction::Outgoing,
|
||||
}
|
||||
}
|
||||
|
||||
fn includes(&self, direction: Direction) -> bool {
|
||||
*self == Direction::Any || *self == direction
|
||||
}
|
||||
}
|
||||
|
||||
/// Whose mail a journal takes (JR-9): everyone, or people reached through
|
||||
/// their account, domain, group or tenant. Ids are in the JMAP form.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Scope {
|
||||
#[serde(default)]
|
||||
pub everyone: bool,
|
||||
#[serde(default, with = "jmap_ids")]
|
||||
pub accounts: Vec<u32>,
|
||||
#[serde(default, with = "jmap_ids")]
|
||||
pub groups: Vec<u32>,
|
||||
#[serde(default, with = "jmap_ids")]
|
||||
pub domains: Vec<u32>,
|
||||
#[serde(default, with = "jmap_ids")]
|
||||
pub tenants: Vec<u32>,
|
||||
}
|
||||
|
||||
impl Scope {
|
||||
fn lists(&self) -> [&Vec<u32>; 4] {
|
||||
[&self.accounts, &self.groups, &self.domains, &self.tenants]
|
||||
}
|
||||
|
||||
/// Whether this scope reaches one person here.
|
||||
pub fn covers(&self, member: &Member) -> bool {
|
||||
self.everyone
|
||||
|| self.accounts.contains(&member.account)
|
||||
|| member.domains.iter().any(|d| self.domains.contains(d))
|
||||
|| member.groups.iter().any(|g| self.groups.contains(g))
|
||||
|| member.tenant.is_some_and(|t| self.tenants.contains(&t))
|
||||
}
|
||||
}
|
||||
|
||||
/// A journal (JR-9): what it takes, and how long its entries are kept.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Journal {
|
||||
#[serde(default)]
|
||||
pub id: u32,
|
||||
pub name: String,
|
||||
#[serde(default)]
|
||||
pub description: String,
|
||||
#[serde(default)]
|
||||
pub enabled: bool,
|
||||
pub direction: Direction,
|
||||
pub scope: Scope,
|
||||
/// How long an entry this journal writes is kept. An entry keeps the
|
||||
/// retention it was written with (JR-12).
|
||||
pub retention_days: u32,
|
||||
#[serde(default)]
|
||||
pub created_by: String,
|
||||
#[serde(default)]
|
||||
pub created_at: u64,
|
||||
#[serde(default)]
|
||||
pub updated_at: u64,
|
||||
}
|
||||
|
||||
/// Why a journal was refused: the property, and what to do.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Invalid {
|
||||
pub property: &'static str,
|
||||
pub reason: String,
|
||||
}
|
||||
|
||||
fn invalid(property: &'static str, reason: impl Into<String>) -> Result<(), Invalid> {
|
||||
Err(Invalid {
|
||||
property,
|
||||
reason: reason.into(),
|
||||
})
|
||||
}
|
||||
|
||||
impl Journal {
|
||||
pub fn validate(&self) -> Result<(), Invalid> {
|
||||
if self.name.trim().is_empty() {
|
||||
return invalid("name", "Give the journal a name.");
|
||||
}
|
||||
if self.name.len() > 200 || self.description.len() > 2_000 {
|
||||
return invalid("name", "The name or description is too long.");
|
||||
}
|
||||
if !(MIN_RETENTION_DAYS..=MAX_RETENTION_DAYS).contains(&self.retention_days) {
|
||||
return invalid(
|
||||
"retentionDays",
|
||||
format!("Keep entries between {MIN_RETENTION_DAYS} and {MAX_RETENTION_DAYS} days."),
|
||||
);
|
||||
}
|
||||
let chosen = self.scope.lists().iter().any(|list| !list.is_empty());
|
||||
if self.scope.everyone == chosen {
|
||||
return invalid(
|
||||
"scope",
|
||||
"Journal everyone, or choose accounts, groups, domains or tenants; not both.",
|
||||
);
|
||||
}
|
||||
if self.scope.lists().iter().any(|list| list.len() > MAX_LIST) {
|
||||
return invalid("scope", format!("Choose at most {MAX_LIST} of each."));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Whether this journal takes a message going `direction` with these
|
||||
/// people here on either side.
|
||||
pub fn takes(&self, direction: Direction, members: &[Member]) -> bool {
|
||||
self.enabled
|
||||
&& self.direction.includes(direction)
|
||||
&& (self.scope.everyone || members.iter().any(|m| self.scope.covers(m)))
|
||||
}
|
||||
}
|
||||
|
||||
/// A value stored as JSON.
|
||||
pub(crate) struct Json<T>(pub T);
|
||||
|
||||
impl<T: SerdeSerialize> Serialize for Json<T> {
|
||||
fn serialize(&self) -> trc::Result<Vec<u8>> {
|
||||
serde_json::to_vec(&self.0).map_err(|err| {
|
||||
trc::StoreEvent::UnexpectedError
|
||||
.into_err()
|
||||
.details("Failed to serialize a journal record")
|
||||
.reason(err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: DeserializeOwned + Sync + Send> Deserialize for Json<T> {
|
||||
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
|
||||
serde_json::from_slice(bytes).map(Json).map_err(|err| {
|
||||
trc::StoreEvent::DataCorruption
|
||||
.into_err()
|
||||
.details("Invalid journal record")
|
||||
.reason(err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn class(id: u32) -> ValueClass {
|
||||
let mut key = Vec::with_capacity(6);
|
||||
key.push(FEATURE);
|
||||
key.push(KIND_JOURNAL);
|
||||
key.extend_from_slice(&id.to_be_bytes());
|
||||
ValueClass::Any(AnyClass {
|
||||
subspace: SUBSPACE_INBUXA,
|
||||
key,
|
||||
})
|
||||
}
|
||||
|
||||
fn key(id: u32) -> ValueKey<ValueClass> {
|
||||
ValueKey::from(class(id))
|
||||
}
|
||||
|
||||
pub async fn get(data: &Store, id: u32) -> trc::Result<Option<Journal>> {
|
||||
Ok(data
|
||||
.get_value::<Json<Journal>>(key(id))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.map(|Json(journal)| journal))
|
||||
}
|
||||
|
||||
/// Every journal, oldest first.
|
||||
pub async fn all(data: &Store) -> trc::Result<Vec<Journal>> {
|
||||
let mut journals = Vec::new();
|
||||
data.iterate(IterateParams::new(key(0), key(u32::MAX)), |_, value| {
|
||||
if let Ok(Json(journal)) = Json::<Journal>::deserialize(value) {
|
||||
journals.push(journal);
|
||||
}
|
||||
Ok(true)
|
||||
})
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
journals.sort_by_key(|journal| journal.id);
|
||||
Ok(journals)
|
||||
}
|
||||
|
||||
/// Writes a new journal under the next free id, which it returns.
|
||||
pub async fn create(data: &Store, journal: &Journal) -> trc::Result<u32> {
|
||||
let mut attempt = 0;
|
||||
loop {
|
||||
attempt += 1;
|
||||
let id = all(data).await?.iter().map(|j| j.id).max().unwrap_or(0) + 1;
|
||||
let stored = Journal {
|
||||
id,
|
||||
..journal.clone()
|
||||
};
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.assert_value(class(id), AssertValue::None);
|
||||
batch.set(class(id), Json(&stored).serialize()?);
|
||||
match data.write(batch.build_all()).await {
|
||||
Ok(_) => {
|
||||
invalidate();
|
||||
return Ok(id);
|
||||
}
|
||||
Err(err)
|
||||
if attempt < CREATE_ATTEMPTS
|
||||
&& matches!(
|
||||
err.as_ref(),
|
||||
trc::EventType::Store(trc::StoreEvent::AssertValueFailed)
|
||||
) => {}
|
||||
Err(err) => return Err(err.caused_by(trc::location!())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Replaces a stored journal (same id).
|
||||
pub async fn update(data: &Store, journal: &Journal) -> trc::Result<()> {
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.set(class(journal.id), Json(journal).serialize()?);
|
||||
data.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
invalidate();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Removes a journal. Its entries stay, each until its own time.
|
||||
pub async fn delete(data: &Store, id: u32) -> trc::Result<()> {
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.clear(class(id));
|
||||
data.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
invalidate();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// How long a node keeps its copy of the journals before reading them again.
|
||||
pub const TTL: Duration = Duration::from_secs(30);
|
||||
|
||||
type Cached = Option<(Instant, Arc<Vec<Journal>>)>;
|
||||
static CACHE: RwLock<Cached> = RwLock::new(None);
|
||||
|
||||
/// Forgets this node's copy, so the next message reads the journals again.
|
||||
pub fn invalidate() {
|
||||
if let Ok(mut cache) = CACHE.write() {
|
||||
*cache = None;
|
||||
}
|
||||
}
|
||||
|
||||
/// The enabled journals, from this node's copy (refreshed every [`TTL`]).
|
||||
pub async fn enabled(data: &Store) -> trc::Result<Arc<Vec<Journal>>> {
|
||||
if let Ok(cache) = CACHE.read()
|
||||
&& let Some((at, journals)) = cache.as_ref()
|
||||
&& at.elapsed() < TTL
|
||||
{
|
||||
return Ok(journals.clone());
|
||||
}
|
||||
let journals = Arc::new(
|
||||
all(data)
|
||||
.await?
|
||||
.into_iter()
|
||||
.filter(|journal| journal.enabled)
|
||||
.collect::<Vec<_>>(),
|
||||
);
|
||||
if let Ok(mut cache) = CACHE.write() {
|
||||
*cache = Some((Instant::now(), journals.clone()));
|
||||
}
|
||||
Ok(journals)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn journal(scope: Scope) -> Journal {
|
||||
Journal {
|
||||
id: 1,
|
||||
name: "Finance".into(),
|
||||
description: String::new(),
|
||||
enabled: true,
|
||||
direction: Direction::Any,
|
||||
scope,
|
||||
retention_days: 365,
|
||||
created_by: String::new(),
|
||||
created_at: 0,
|
||||
updated_at: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn member(account: u32, groups: Vec<u32>) -> Member {
|
||||
Member {
|
||||
account,
|
||||
domains: vec![1],
|
||||
groups,
|
||||
tenant: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scope_is_everyone_or_chosen() {
|
||||
assert!(
|
||||
journal(Scope {
|
||||
everyone: true,
|
||||
..Default::default()
|
||||
})
|
||||
.validate()
|
||||
.is_ok()
|
||||
);
|
||||
assert!(journal(Scope::default()).validate().is_err());
|
||||
let both = Scope {
|
||||
everyone: true,
|
||||
groups: vec![4],
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(journal(both).validate().unwrap_err().property, "scope");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retention_has_bounds() {
|
||||
let mut j = journal(Scope {
|
||||
everyone: true,
|
||||
..Default::default()
|
||||
});
|
||||
j.retention_days = 29;
|
||||
assert_eq!(j.validate().unwrap_err().property, "retentionDays");
|
||||
j.retention_days = 3651;
|
||||
assert!(j.validate().is_err());
|
||||
j.retention_days = 3650;
|
||||
assert!(j.validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn takes_by_direction_and_member() {
|
||||
let mut j = journal(Scope {
|
||||
groups: vec![7],
|
||||
..Default::default()
|
||||
});
|
||||
assert!(j.takes(Direction::Outgoing, &[member(3, vec![7])]));
|
||||
assert!(!j.takes(Direction::Outgoing, &[member(3, vec![8])]));
|
||||
assert!(!j.takes(Direction::Outgoing, &[]));
|
||||
j.direction = Direction::Incoming;
|
||||
assert!(!j.takes(Direction::Outgoing, &[member(3, vec![7])]));
|
||||
j.enabled = false;
|
||||
assert!(!j.takes(Direction::Incoming, &[member(3, vec![7])]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn directions() {
|
||||
assert_eq!(Direction::of(true, true, true), Direction::Outgoing);
|
||||
assert_eq!(Direction::of(true, false, true), Direction::Internal);
|
||||
assert_eq!(Direction::of(false, false, true), Direction::Incoming);
|
||||
assert_eq!(Direction::of(false, true, true), Direction::Incoming);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scope_ids_are_jmap_ids() {
|
||||
let scope: Scope = serde_json::from_str(r#"{"groups":["b"],"tenants":[7]}"#).unwrap();
|
||||
assert_eq!(scope.groups, vec![1]);
|
||||
assert_eq!(scope.tenants, vec![7]);
|
||||
assert_eq!(
|
||||
serde_json::to_value(&scope).unwrap()["tenants"],
|
||||
serde_json::json!(["h"])
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,359 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! The journal report (JR-3, JR-4): a message whose first part lists the
|
||||
//! envelope, one field a line, and whose second part is the message as it
|
||||
//! was queued, byte for byte, as `message/rfc822`. Field names are fixed
|
||||
//! English: a report is a record, and scripts read it.
|
||||
|
||||
use super::Direction;
|
||||
use mail_builder::headers::{Header, date::Date, text::Text};
|
||||
use mail_parser::MessageParser;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
/// One envelope recipient, with the address it was given as (a list's, for
|
||||
/// the list's members).
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Recipient {
|
||||
pub address: String,
|
||||
pub orcpt: Option<String>,
|
||||
}
|
||||
|
||||
/// What the queue knows about a message.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Envelope<'x> {
|
||||
pub sender: &'x str,
|
||||
pub authenticated: bool,
|
||||
pub recipients: &'x [Recipient],
|
||||
pub queue_id: u64,
|
||||
/// Seconds.
|
||||
pub received: u64,
|
||||
pub direction: Direction,
|
||||
pub held: bool,
|
||||
}
|
||||
|
||||
/// What a report says, besides the envelope's own fields.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct Fields {
|
||||
pub subject: String,
|
||||
pub message_id: String,
|
||||
pub to: Vec<String>,
|
||||
pub cc: Vec<String>,
|
||||
/// Envelope recipients in neither To nor Cc, nor reached through a list.
|
||||
pub bcc: Vec<String>,
|
||||
/// A list's address, and its members among the recipients.
|
||||
pub expanded: Vec<(String, Vec<String>)>,
|
||||
}
|
||||
|
||||
/// One line's worth of a value: no line breaks, no control characters.
|
||||
fn line(value: &str) -> String {
|
||||
value
|
||||
.chars()
|
||||
.map(|c| if c.is_control() { ' ' } else { c })
|
||||
.collect::<String>()
|
||||
.trim()
|
||||
.to_string()
|
||||
}
|
||||
|
||||
/// The address an ORCPT names, without its `rfc822;` type.
|
||||
fn orcpt_address(orcpt: &str) -> String {
|
||||
let orcpt = orcpt.trim();
|
||||
let bare = match orcpt.split_once(';') {
|
||||
Some((kind, address)) if kind.eq_ignore_ascii_case("rfc822") => address,
|
||||
_ => orcpt,
|
||||
};
|
||||
bare.trim().to_lowercase()
|
||||
}
|
||||
|
||||
/// Sorts the envelope's recipients by how they were addressed.
|
||||
pub fn fields(envelope: &Envelope<'_>, original: &[u8]) -> Fields {
|
||||
let parsed = MessageParser::default().parse_headers(original);
|
||||
let headed = |which: Option<&mail_parser::Address<'_>>| -> Vec<String> {
|
||||
which
|
||||
.map(|list| {
|
||||
list.iter()
|
||||
.filter_map(|addr| addr.address())
|
||||
.map(|address| address.to_lowercase())
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default()
|
||||
};
|
||||
let (subject, message_id, header_to, header_cc) = match &parsed {
|
||||
Some(message) => (
|
||||
message.subject().map(line).unwrap_or_default(),
|
||||
message
|
||||
.message_id()
|
||||
.map(|id| format!("<{}>", line(id)))
|
||||
.unwrap_or_default(),
|
||||
headed(message.to()),
|
||||
headed(message.cc()),
|
||||
),
|
||||
None => Default::default(),
|
||||
};
|
||||
|
||||
let mut fields = Fields {
|
||||
subject,
|
||||
message_id,
|
||||
..Default::default()
|
||||
};
|
||||
for rcpt in envelope.recipients {
|
||||
let address = rcpt.address.to_lowercase();
|
||||
let via = rcpt
|
||||
.orcpt
|
||||
.as_deref()
|
||||
.map(orcpt_address)
|
||||
.filter(|via| !via.is_empty() && *via != address);
|
||||
if header_to.contains(&address) {
|
||||
fields.to.push(line(&rcpt.address));
|
||||
} else if header_cc.contains(&address) {
|
||||
fields.cc.push(line(&rcpt.address));
|
||||
} else if let Some(via) = via {
|
||||
match fields.expanded.iter_mut().find(|(list, _)| *list == via) {
|
||||
Some((_, members)) => members.push(line(&rcpt.address)),
|
||||
None => fields
|
||||
.expanded
|
||||
.push((line(&via), vec![line(&rcpt.address)])),
|
||||
}
|
||||
} else {
|
||||
fields.bcc.push(line(&rcpt.address));
|
||||
}
|
||||
}
|
||||
fields
|
||||
}
|
||||
|
||||
/// The report's first part.
|
||||
pub fn text(envelope: &Envelope<'_>, fields: &Fields) -> String {
|
||||
let mut out = String::new();
|
||||
let mut field = |name: &str, value: &str| {
|
||||
if !value.is_empty() {
|
||||
out.push_str(name);
|
||||
out.push_str(": ");
|
||||
out.push_str(value);
|
||||
out.push_str("\r\n");
|
||||
}
|
||||
};
|
||||
let sender = if envelope.sender.is_empty() {
|
||||
"<>".to_string()
|
||||
} else {
|
||||
line(envelope.sender)
|
||||
};
|
||||
field("Sender", &sender);
|
||||
field(
|
||||
"Authenticated",
|
||||
if envelope.authenticated { "yes" } else { "no" },
|
||||
);
|
||||
field("Subject", &fields.subject);
|
||||
field("Message-ID", &fields.message_id);
|
||||
field("Queue ID", &format!("{:x}", envelope.queue_id));
|
||||
field(
|
||||
"Received",
|
||||
&mail_parser::DateTime::from_timestamp(envelope.received as i64).to_rfc3339(),
|
||||
);
|
||||
field("Direction", envelope.direction.as_str());
|
||||
field("To", &fields.to.join(", "));
|
||||
field("Cc", &fields.cc.join(", "));
|
||||
field("Bcc", &fields.bcc.join(", "));
|
||||
for (list, members) in &fields.expanded {
|
||||
field("Expanded", &format!("{list} -> {}", members.join(", ")));
|
||||
}
|
||||
if envelope.held {
|
||||
field("Held for review", "yes");
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn hex(bytes: &[u8]) -> String {
|
||||
bytes.iter().map(|b| format!("{b:02x}")).collect()
|
||||
}
|
||||
|
||||
/// Whether a message can travel as 8bit: no NULs, no line past 998 bytes.
|
||||
fn fits_8bit(message: &[u8]) -> bool {
|
||||
!message.contains(&0) && message.split(|b| *b == b'\n').all(|l| l.len() <= 998)
|
||||
}
|
||||
|
||||
/// The whole report: headers, the fields, then the original untouched.
|
||||
/// `from` is the address the report is from; `host` names the server in its
|
||||
/// Message-ID.
|
||||
pub fn build(
|
||||
envelope: &Envelope<'_>,
|
||||
original: &[u8],
|
||||
from: &str,
|
||||
host: &str,
|
||||
) -> (Vec<u8>, Fields) {
|
||||
let fields = fields(envelope, original);
|
||||
let body = text(envelope, &fields);
|
||||
// A boundary that can't occur in the original
|
||||
let mut boundary = format!("journal-{}", &hex(&Sha256::digest(original))[..32]);
|
||||
while original
|
||||
.windows(boundary.len())
|
||||
.any(|window| window == boundary.as_bytes())
|
||||
{
|
||||
boundary.push('x');
|
||||
}
|
||||
|
||||
let mut out: Vec<u8> = Vec::with_capacity(original.len() + body.len() + 1024);
|
||||
out.extend_from_slice(format!("From: Journal <{}>\r\n", line(from)).as_bytes());
|
||||
out.extend_from_slice(b"Date: ");
|
||||
out.extend_from_slice(Date::new(envelope.received as i64).to_rfc822().as_bytes());
|
||||
out.extend_from_slice(b"\r\n");
|
||||
out.extend_from_slice(b"Subject: ");
|
||||
let subject = if fields.subject.is_empty() {
|
||||
"Journal report".to_string()
|
||||
} else {
|
||||
format!("Journal report: {}", fields.subject)
|
||||
};
|
||||
Text::new(subject).write_header(&mut out, "Subject: ".len());
|
||||
out.extend_from_slice(
|
||||
format!(
|
||||
"Message-ID: <journal.{:x}.{}@{}>\r\n",
|
||||
envelope.queue_id,
|
||||
envelope.received,
|
||||
line(host)
|
||||
)
|
||||
.as_bytes(),
|
||||
);
|
||||
out.extend_from_slice(format!("X-Inbuxa-Journal: {:x}\r\n", envelope.queue_id).as_bytes());
|
||||
out.extend_from_slice(b"MIME-Version: 1.0\r\n");
|
||||
out.extend_from_slice(
|
||||
format!("Content-Type: multipart/mixed; boundary=\"{boundary}\"\r\n\r\n").as_bytes(),
|
||||
);
|
||||
out.extend_from_slice(format!("--{boundary}\r\n").as_bytes());
|
||||
out.extend_from_slice(
|
||||
b"Content-Type: text/plain; charset=utf-8\r\nContent-Transfer-Encoding: 8bit\r\n\r\n",
|
||||
);
|
||||
out.extend_from_slice(body.as_bytes());
|
||||
out.extend_from_slice(format!("\r\n--{boundary}\r\n").as_bytes());
|
||||
out.extend_from_slice(b"Content-Type: message/rfc822\r\n");
|
||||
out.extend_from_slice(b"Content-Disposition: attachment; filename=\"original.eml\"\r\n");
|
||||
out.extend_from_slice(if fits_8bit(original) {
|
||||
b"Content-Transfer-Encoding: 8bit\r\n\r\n".as_slice()
|
||||
} else {
|
||||
b"Content-Transfer-Encoding: binary\r\n\r\n".as_slice()
|
||||
});
|
||||
out.extend_from_slice(original);
|
||||
// The line break before a boundary belongs to the boundary: the
|
||||
// original keeps its own last one
|
||||
out.extend_from_slice(format!("\r\n--{boundary}--\r\n").as_bytes());
|
||||
(out, fields)
|
||||
}
|
||||
|
||||
/// Where the original starts and ends inside a report [`build`] made.
|
||||
pub fn original(report: &[u8]) -> Option<&[u8]> {
|
||||
let parsed = MessageParser::default().parse(report)?;
|
||||
let part = parsed.attachment(0)?;
|
||||
let start = part.raw_body_offset() as usize;
|
||||
let end = part.raw_end_offset() as usize;
|
||||
report.get(start..end)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const ORIGINAL: &[u8] = b"From: [email protected]\r\n\
|
||||
To: Bank <[email protected]>\r\n\
|
||||
Cc: [email protected]\r\n\
|
||||
Subject: Q3 figures\r\n\
|
||||
Message-ID: <[email protected]>\r\n\
|
||||
\r\n\
|
||||
The figures.\r\n";
|
||||
|
||||
fn rcpt(address: &str, orcpt: Option<&str>) -> Recipient {
|
||||
Recipient {
|
||||
address: address.into(),
|
||||
orcpt: orcpt.map(Into::into),
|
||||
}
|
||||
}
|
||||
|
||||
fn envelope(recipients: &[Recipient]) -> Envelope<'_> {
|
||||
Envelope {
|
||||
sender: "[email protected]",
|
||||
authenticated: true,
|
||||
recipients,
|
||||
queue_id: 0x1a2b,
|
||||
received: 1_790_000_000,
|
||||
direction: Direction::Outgoing,
|
||||
held: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recipients_sorted_by_how_they_were_addressed() {
|
||||
let recipients = [
|
||||
rcpt("[email protected]", None),
|
||||
rcpt("[email protected]", Some("rfc822;[email protected]")),
|
||||
rcpt("[email protected]", None),
|
||||
rcpt("[email protected]", Some("[email protected]")),
|
||||
rcpt("[email protected]", Some("rfc822;[email protected]")),
|
||||
];
|
||||
let fields = fields(&envelope(&recipients), ORIGINAL);
|
||||
assert_eq!(fields.subject, "Q3 figures");
|
||||
assert_eq!(fields.message_id, "<[email protected]>");
|
||||
assert_eq!(fields.to, vec!["[email protected]"]);
|
||||
assert_eq!(fields.cc, vec!["[email protected]"]);
|
||||
assert_eq!(fields.bcc, vec!["[email protected]"]);
|
||||
assert_eq!(
|
||||
fields.expanded,
|
||||
vec![(
|
||||
"[email protected]".to_string(),
|
||||
vec![
|
||||
"[email protected]".to_string(),
|
||||
"[email protected]".to_string()
|
||||
]
|
||||
)]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn report_carries_the_original_untouched() {
|
||||
let recipients = [
|
||||
rcpt("[email protected]", None),
|
||||
rcpt("[email protected]", None),
|
||||
];
|
||||
let (report, _) = build(
|
||||
&envelope(&recipients),
|
||||
ORIGINAL,
|
||||
"[email protected]",
|
||||
"mx.example.com",
|
||||
);
|
||||
let text = String::from_utf8_lossy(&report);
|
||||
assert!(text.contains("Sender: [email protected]\r\n"));
|
||||
assert!(text.contains("Bcc: [email protected]\r\n"));
|
||||
assert!(text.contains("Queue ID: 1a2b\r\n"));
|
||||
assert!(text.contains("Direction: outgoing\r\n"));
|
||||
assert!(text.contains("Subject: Journal report: Q3 figures\r\n"));
|
||||
assert!(!text.contains("Held for review"));
|
||||
assert_eq!(original(&report), Some(ORIGINAL));
|
||||
let unterminated = &ORIGINAL[..ORIGINAL.len() - 2];
|
||||
let (report, _) = build(
|
||||
&envelope(&recipients),
|
||||
unterminated,
|
||||
"[email protected]",
|
||||
"mx.example.com",
|
||||
);
|
||||
assert_eq!(original(&report), Some(unterminated));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn values_stay_on_one_line() {
|
||||
let recipients = [rcpt("[email protected]", None)];
|
||||
let mut env = envelope(&recipients);
|
||||
env.sender = "[email protected]\r\nBcc: [email protected]";
|
||||
env.held = true;
|
||||
let body = text(&env, &Fields::default());
|
||||
assert_eq!(body.matches("\r\n").count(), body.lines().count());
|
||||
assert!(body.contains("Sender: [email protected] Bcc: [email protected]\r\n"));
|
||||
assert!(body.contains("Held for review: yes\r\n"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_sender_is_shown_as_such() {
|
||||
let recipients = [rcpt("[email protected]", None)];
|
||||
let mut env = envelope(&recipients);
|
||||
env.sender = "";
|
||||
assert!(text(&env, &Fields::default()).starts_with("Sender: <>\r\n"));
|
||||
}
|
||||
}
|
||||
@@ -22,7 +22,9 @@ pub mod ai;
|
||||
pub mod audit;
|
||||
pub mod branding;
|
||||
pub mod hold;
|
||||
pub mod journal;
|
||||
pub mod lock;
|
||||
pub mod mailflow;
|
||||
pub mod masked_email;
|
||||
pub mod privacy;
|
||||
pub mod security;
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! The compiled rules, kept per node so a message doesn't read the store.
|
||||
//! A change made on this node applies at once; one made on another node
|
||||
//! within [`TTL`], when the copy here is next refreshed.
|
||||
|
||||
use super::{engine::Compiled, rules};
|
||||
use std::{
|
||||
sync::{Arc, RwLock},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use store::Store;
|
||||
|
||||
/// How long a node keeps its copy before reading the rules again.
|
||||
pub const TTL: Duration = Duration::from_secs(30);
|
||||
|
||||
static CACHE: RwLock<Option<(Instant, Arc<Compiled>)>> = RwLock::new(None);
|
||||
|
||||
/// Forgets the copy, so the next message reads the rules again.
|
||||
pub fn invalidate() {
|
||||
if let Ok(mut cache) = CACHE.write() {
|
||||
*cache = None;
|
||||
}
|
||||
}
|
||||
|
||||
/// The enabled rules, compiled. A rule that no longer compiles is left out
|
||||
/// and reported, once per refresh.
|
||||
pub async fn compiled(data: &Store) -> trc::Result<Arc<Compiled>> {
|
||||
if let Ok(cache) = CACHE.read()
|
||||
&& let Some((at, compiled)) = cache.as_ref()
|
||||
&& at.elapsed() < TTL
|
||||
{
|
||||
return Ok(compiled.clone());
|
||||
}
|
||||
let (compiled, skipped) = Compiled::new(&rules::all(data).await?);
|
||||
for (id, reason) in skipped {
|
||||
trc::event!(
|
||||
Store(trc::StoreEvent::DataCorruption),
|
||||
Id = u64::from(id),
|
||||
Reason = reason,
|
||||
Details = "Mail rule skipped: it no longer compiles"
|
||||
);
|
||||
}
|
||||
let compiled = Arc::new(compiled);
|
||||
if let Ok(mut cache) = CACHE.write() {
|
||||
*cache = Some((Instant::now(), compiled.clone()));
|
||||
}
|
||||
Ok(compiled)
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! African identifiers (§2.3): South Africa's ID number.
|
||||
|
||||
use super::{Detector, Findings, Region, Strength, checks, valid_short_date};
|
||||
use regex::Regex;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub static DETECTORS: &[Detector] = &[Detector::new(
|
||||
"za-id",
|
||||
"South Africa: ID number",
|
||||
Region::Africa,
|
||||
Strength::Checked,
|
||||
za_id,
|
||||
)];
|
||||
|
||||
/// Birth date `YYMMDD`, four digits, citizenship (0, 1 or 2), 8 or 9, a Luhn
|
||||
/// check digit. The date and the two fixed digits make it strong enough to
|
||||
/// count alone.
|
||||
static ZA_ID: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r"\b(\d{2})(\d{2})(\d{2})\d{4}[012][89]\d\b").expect("detector pattern")
|
||||
});
|
||||
|
||||
fn za_id(text: &str, findings: &mut Findings) {
|
||||
for c in ZA_ID.captures_iter(text) {
|
||||
let n = &c[0];
|
||||
let num = |s: &str| s.parse::<u32>().unwrap_or(0);
|
||||
if valid_short_date(num(&c[1]), num(&c[2]), num(&c[3])) && checks::luhn(n) {
|
||||
findings.insert(n);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::mailflow::detectors::by_id;
|
||||
|
||||
#[test]
|
||||
fn south_africa() {
|
||||
let detector = by_id("za-id").unwrap();
|
||||
assert_eq!(detector.count("ID 8001015009087"), 1);
|
||||
assert_eq!(detector.count("8001015009088"), 0);
|
||||
assert_eq!(detector.count("8013015009087"), 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! Identifiers from the Americas outside the US and Canada (§2.3): Brazil's
|
||||
//! CPF and CNPJ, and Mexico's CURP.
|
||||
|
||||
use super::{Detector, Findings, Region, Strength, digit_values, valid_short_date, word_near};
|
||||
use regex::Regex;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub static DETECTORS: &[Detector] = &[
|
||||
Detector::new(
|
||||
"br-cpf",
|
||||
"Brazil: CPF",
|
||||
Region::Americas,
|
||||
Strength::Checked,
|
||||
br_cpf,
|
||||
),
|
||||
Detector::new(
|
||||
"br-cnpj",
|
||||
"Brazil: CNPJ",
|
||||
Region::Americas,
|
||||
Strength::Checked,
|
||||
br_cnpj,
|
||||
),
|
||||
Detector::new(
|
||||
"mx-curp",
|
||||
"Mexico: CURP",
|
||||
Region::Americas,
|
||||
Strength::Checked,
|
||||
mx_curp,
|
||||
),
|
||||
];
|
||||
|
||||
fn re(pattern: &str) -> Regex {
|
||||
Regex::new(pattern).expect("detector pattern")
|
||||
}
|
||||
|
||||
/// Brazil's mod 11 check digit over `digits` with `weights`.
|
||||
fn br_check(digits: &[u32], weights: &[u32]) -> u32 {
|
||||
match digits.iter().zip(weights).map(|(a, w)| a * w).sum::<u32>() % 11 {
|
||||
0 | 1 => 0,
|
||||
r => 11 - r,
|
||||
}
|
||||
}
|
||||
|
||||
/// `111.444.777-35`, or eleven bare digits.
|
||||
static CPF: LazyLock<Regex> = LazyLock::new(|| re(r"\b\d{3}(\.?)\d{3}(\.?)\d{3}(-?)\d{2}\b"));
|
||||
|
||||
pub fn cpf_valid(n: &str) -> bool {
|
||||
let d = digit_values(n);
|
||||
// A run of one digit passes the arithmetic but is never issued
|
||||
d.len() == 11
|
||||
&& d.iter().any(|x| *x != d[0])
|
||||
&& br_check(&d[..9], &[10, 9, 8, 7, 6, 5, 4, 3, 2]) == d[9]
|
||||
&& br_check(&d[..10], &[11, 10, 9, 8, 7, 6, 5, 4, 3, 2]) == d[10]
|
||||
}
|
||||
|
||||
const CPF_WORDS: &[&str] = &[
|
||||
"cpf",
|
||||
"cadastro de pessoas físicas",
|
||||
"cadastro de pessoa física",
|
||||
];
|
||||
|
||||
fn br_cpf(text: &str, findings: &mut Findings) {
|
||||
for c in CPF.captures_iter(text) {
|
||||
let whole = c.get(0).unwrap();
|
||||
let written = &c[1] == "." && &c[2] == "." && &c[3] == "-";
|
||||
let n: String = whole
|
||||
.as_str()
|
||||
.chars()
|
||||
.filter(char::is_ascii_digit)
|
||||
.collect();
|
||||
if cpf_valid(&n) && (written || word_near(text, whole.start(), whole.end(), CPF_WORDS)) {
|
||||
findings.insert(n);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `11.222.333/0001-81`, or fourteen bare digits.
|
||||
static CNPJ: LazyLock<Regex> =
|
||||
LazyLock::new(|| re(r"\b\d{2}(\.?)\d{3}(\.?)\d{3}(/?)\d{4}(-?)\d{2}\b"));
|
||||
|
||||
pub fn cnpj_valid(n: &str) -> bool {
|
||||
let d = digit_values(n);
|
||||
d.len() == 14
|
||||
&& d.iter().any(|x| *x != d[0])
|
||||
&& br_check(&d[..12], &[5, 4, 3, 2, 9, 8, 7, 6, 5, 4, 3, 2]) == d[12]
|
||||
&& br_check(&d[..13], &[6, 5, 4, 3, 2, 9, 8, 7, 6, 5, 4, 3, 2]) == d[13]
|
||||
}
|
||||
|
||||
const CNPJ_WORDS: &[&str] = &["cnpj", "cadastro nacional da pessoa jurídica"];
|
||||
|
||||
fn br_cnpj(text: &str, findings: &mut Findings) {
|
||||
for c in CNPJ.captures_iter(text) {
|
||||
let whole = c.get(0).unwrap();
|
||||
let written = &c[1] == "." && &c[2] == "." && &c[3] == "/" && &c[4] == "-";
|
||||
let n: String = whole
|
||||
.as_str()
|
||||
.chars()
|
||||
.filter(char::is_ascii_digit)
|
||||
.collect();
|
||||
if cnpj_valid(&n) && (written || word_near(text, whole.start(), whole.end(), CNPJ_WORDS)) {
|
||||
findings.insert(n);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Four letters, the birth date, sex (H, M or X), the state, three
|
||||
/// consonants, a character that tells the century apart, the check digit.
|
||||
static CURP: LazyLock<Regex> = LazyLock::new(|| {
|
||||
re(r"(?i)\b[A-Z]{4}(\d{2})(\d{2})(\d{2})[HMX][A-Z]{2}[B-DF-HJ-NP-TV-Z]{3}[A-Z0-9]\d\b")
|
||||
});
|
||||
|
||||
/// RENAPO's check: each character's place in `0-9 A-N Ñ O-Z`, weighted 18
|
||||
/// down to 2; the digit is 10 minus the sum mod 10 (10 becomes 0).
|
||||
pub fn curp_valid(curp: &str) -> bool {
|
||||
const ALPHABET: &str = "0123456789ABCDEFGHIJKLMNÑOPQRSTUVWXYZ";
|
||||
let mut sum = 0u32;
|
||||
for (i, c) in curp.chars().take(17).enumerate() {
|
||||
let Some(value) = ALPHABET.chars().position(|a| a == c) else {
|
||||
return false;
|
||||
};
|
||||
sum += value as u32 * (18 - i as u32);
|
||||
}
|
||||
curp.chars().nth(17).and_then(|c| c.to_digit(10)) == Some((10 - sum % 10) % 10)
|
||||
}
|
||||
|
||||
fn mx_curp(text: &str, findings: &mut Findings) {
|
||||
for c in CURP.captures_iter(text) {
|
||||
let curp = c[0].to_ascii_uppercase();
|
||||
if valid_short_date(num(&c[1]), num(&c[2]), num(&c[3])) && curp_valid(&curp) {
|
||||
findings.insert(curp);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn num(s: &str) -> u32 {
|
||||
s.parse().unwrap_or(0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::mailflow::detectors::by_id;
|
||||
|
||||
fn count(id: &str, text: &str) -> usize {
|
||||
by_id(id).unwrap().count(text)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn brazil() {
|
||||
assert_eq!(count("br-cpf", "CPF 111.444.777-35"), 1);
|
||||
assert_eq!(count("br-cpf", "111.444.777-36"), 0);
|
||||
assert_eq!(count("br-cpf", "pedido 11144477735"), 0);
|
||||
assert_eq!(count("br-cpf", "cpf: 11144477735"), 1);
|
||||
assert_eq!(count("br-cpf", "CPF 111.111.111-11"), 0);
|
||||
assert_eq!(count("br-cnpj", "11.222.333/0001-81"), 1);
|
||||
assert_eq!(count("br-cnpj", "11.222.333/0001-82"), 0);
|
||||
assert_eq!(count("br-cnpj", "CNPJ 11222333000181"), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mexico() {
|
||||
// python-stdnum's documented example
|
||||
assert_eq!(count("mx-curp", "CURP BOXW310820HNERXN09"), 1);
|
||||
assert_eq!(count("mx-curp", "BOXW310820HNERXN08"), 0);
|
||||
assert_eq!(count("mx-curp", "BOXW311320HNERXN09"), 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,511 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! Detectors that aren't tied to one country (§2.3, region "Any").
|
||||
|
||||
use super::{
|
||||
Detector, Findings, Region, Strength, checks, digits, stands_alone, valid_date, word_near,
|
||||
};
|
||||
use regex::Regex;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub static DETECTORS: &[Detector] = &[
|
||||
Detector::new(
|
||||
"payment-card",
|
||||
"Payment card number",
|
||||
Region::Any,
|
||||
Strength::Checked,
|
||||
payment_card,
|
||||
),
|
||||
Detector::new("iban", "IBAN", Region::Any, Strength::Checked, iban),
|
||||
Detector::new(
|
||||
"swift-bic",
|
||||
"SWIFT/BIC code",
|
||||
Region::Any,
|
||||
Strength::NeedsWord,
|
||||
swift_bic,
|
||||
),
|
||||
Detector::new(
|
||||
"email-addresses",
|
||||
"Email addresses",
|
||||
Region::Any,
|
||||
Strength::Checked,
|
||||
email_addresses,
|
||||
),
|
||||
Detector::new(
|
||||
"phone-numbers",
|
||||
"Phone numbers",
|
||||
Region::Any,
|
||||
Strength::NeedsWord,
|
||||
phone_numbers,
|
||||
),
|
||||
Detector::new(
|
||||
"date-of-birth",
|
||||
"Date of birth",
|
||||
Region::Any,
|
||||
Strength::NeedsWord,
|
||||
date_of_birth,
|
||||
),
|
||||
Detector::new(
|
||||
"passport",
|
||||
"Passport number",
|
||||
Region::Any,
|
||||
Strength::NeedsWord,
|
||||
passport,
|
||||
),
|
||||
Detector::new(
|
||||
"private-key",
|
||||
"Private key",
|
||||
Region::Any,
|
||||
Strength::Checked,
|
||||
private_key,
|
||||
),
|
||||
Detector::new(
|
||||
"credentials",
|
||||
"Cloud and service credentials",
|
||||
Region::Any,
|
||||
Strength::Checked,
|
||||
credentials,
|
||||
),
|
||||
];
|
||||
|
||||
fn re(pattern: &str) -> Regex {
|
||||
Regex::new(pattern).expect("detector pattern")
|
||||
}
|
||||
|
||||
// --- Payment cards --------------------------------------------------------
|
||||
|
||||
/// Issuer prefixes (ISO/IEC 7812 IINs) and the lengths each network issues.
|
||||
fn card_network(number: &str) -> bool {
|
||||
let len = number.len();
|
||||
let prefix = |n: usize| number[..n].parse::<u32>().unwrap_or(0);
|
||||
match number.as_bytes()[0] {
|
||||
// Visa
|
||||
b'4' => matches!(len, 13 | 16 | 19),
|
||||
b'5' => {
|
||||
// Mastercard 51–55; Maestro 50, 56–58
|
||||
(51..=55).contains(&prefix(2)) && len == 16
|
||||
|| matches!(prefix(2), 50 | 56..=58) && (12..=19).contains(&len)
|
||||
}
|
||||
// Mastercard 2221–2720
|
||||
b'2' => (2221..=2720).contains(&prefix(4)) && len == 16,
|
||||
b'3' => {
|
||||
// American Express 34, 37; JCB 3528–3589; Diners 300–305, 36, 38, 39
|
||||
matches!(prefix(2), 34 | 37) && len == 15
|
||||
|| (3528..=3589).contains(&prefix(4)) && (16..=19).contains(&len)
|
||||
|| ((300..=305).contains(&prefix(3)) || matches!(prefix(2), 36 | 38 | 39))
|
||||
&& (14..=19).contains(&len)
|
||||
}
|
||||
// Discover 6011, 644–649, 65; UnionPay 62; Maestro 6x
|
||||
b'6' => (12..=19).contains(&len),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_card(number: &str) -> bool {
|
||||
(12..=19).contains(&number.len()) && card_network(number) && checks::luhn(number)
|
||||
}
|
||||
|
||||
static CARD: LazyLock<Regex> = LazyLock::new(|| re(r"\b\d(?:[ -]?\d){11,18}\b"));
|
||||
|
||||
fn payment_card(text: &str, findings: &mut Findings) {
|
||||
for m in CARD.find_iter(text) {
|
||||
if !stands_alone(text, m.start(), m.end()) {
|
||||
continue;
|
||||
}
|
||||
let whole = digits(m.as_str());
|
||||
if is_card(&whole) {
|
||||
findings.insert(whole);
|
||||
continue;
|
||||
}
|
||||
// Two numbers side by side ("4242 4242 4242 4242 2031"): try each
|
||||
// run of whole groups
|
||||
let groups: Vec<String> = m.as_str().split([' ', '-']).map(digits).collect();
|
||||
'runs: for from in 0..groups.len() {
|
||||
let mut number = String::new();
|
||||
for group in &groups[from..] {
|
||||
number.push_str(group);
|
||||
if is_card(&number) {
|
||||
findings.insert(number);
|
||||
break 'runs;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- IBAN -----------------------------------------------------------------
|
||||
|
||||
static IBAN: LazyLock<Regex> =
|
||||
LazyLock::new(|| re(r"\b[A-Za-z]{2}\d{2}(?:[ ]?[A-Za-z0-9]){11,30}"));
|
||||
|
||||
fn iban(text: &str, findings: &mut Findings) {
|
||||
// The pattern can run on into the next words, even the next IBAN: after
|
||||
// each hit, look again from where that IBAN ended
|
||||
let mut from = 0;
|
||||
while let Some(m) = IBAN.find_at(text, from) {
|
||||
from = m.start() + 1;
|
||||
let compact = m.as_str().replace(' ', "").to_ascii_uppercase();
|
||||
let Some(len) = checks::iban_length(&compact[..2]) else {
|
||||
continue;
|
||||
};
|
||||
if compact.len() < len {
|
||||
continue;
|
||||
}
|
||||
// Where the country's length ends in the text, spaces counted
|
||||
let mut seen = 0;
|
||||
let Some(end) = m
|
||||
.as_str()
|
||||
.char_indices()
|
||||
.find(|(_, c)| {
|
||||
if *c != ' ' {
|
||||
seen += 1;
|
||||
}
|
||||
seen == len
|
||||
})
|
||||
.map(|(i, c)| m.start() + i + c.len_utf8())
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let candidate = &compact[..len];
|
||||
if stands_alone(text, m.start(), end) && checks::iban(candidate) {
|
||||
findings.insert(candidate);
|
||||
from = end;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- SWIFT/BIC ------------------------------------------------------------
|
||||
|
||||
static BIC: LazyLock<Regex> =
|
||||
LazyLock::new(|| re(r"\b[A-Z]{4}[A-Z]{2}[A-Z0-9]{2}(?:[A-Z0-9]{3})?\b"));
|
||||
|
||||
const BIC_WORDS: &[&str] = &[
|
||||
"swift",
|
||||
"bic",
|
||||
"swift/bic",
|
||||
"bank",
|
||||
"banque",
|
||||
"bankverbindung",
|
||||
];
|
||||
|
||||
fn swift_bic(text: &str, findings: &mut Findings) {
|
||||
for m in BIC.find_iter(text) {
|
||||
let code = m.as_str();
|
||||
if checks::is_country(&code[4..6]) && word_near(text, m.start(), m.end(), BIC_WORDS) {
|
||||
findings.insert(code);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Contact lists --------------------------------------------------------
|
||||
|
||||
static EMAIL: LazyLock<Regex> =
|
||||
LazyLock::new(|| re(r"(?i)\b[a-z0-9._%+-]+@[a-z0-9-]+(?:\.[a-z0-9-]+)*\.[a-z]{2,}\b"));
|
||||
|
||||
fn email_addresses(text: &str, findings: &mut Findings) {
|
||||
for m in EMAIL.find_iter(text) {
|
||||
findings.insert(m.as_str().to_lowercase());
|
||||
}
|
||||
}
|
||||
|
||||
/// International form: found alone. National form: only with a word.
|
||||
static PHONE_INTL: LazyLock<Regex> = LazyLock::new(|| re(r"\+\d{1,3}(?:[ .-]?\(?\d{1,4}\)?){2,5}"));
|
||||
static PHONE_NATIONAL: LazyLock<Regex> =
|
||||
LazyLock::new(|| re(r"\(?\d{2,4}\)?[ .-]\d{3,4}[ .-]\d{3,4}"));
|
||||
|
||||
const PHONE_WORDS: &[&str] = &[
|
||||
"phone",
|
||||
"tel",
|
||||
"telephone",
|
||||
"mobile",
|
||||
"cell",
|
||||
"fax",
|
||||
"telefon",
|
||||
"téléphone",
|
||||
"teléfono",
|
||||
"telefono",
|
||||
"handy",
|
||||
"portable",
|
||||
"móvil",
|
||||
"cellulare",
|
||||
"mobiel",
|
||||
];
|
||||
|
||||
fn phone_numbers(text: &str, findings: &mut Findings) {
|
||||
let mut international = Vec::new();
|
||||
for m in PHONE_INTL.find_iter(text) {
|
||||
let number = digits(m.as_str());
|
||||
if (8..=15).contains(&number.len()) && stands_alone(text, m.start() + 1, m.end()) {
|
||||
findings.insert(number);
|
||||
international.push(m.range());
|
||||
}
|
||||
}
|
||||
for m in PHONE_NATIONAL.find_iter(text) {
|
||||
let number = digits(m.as_str());
|
||||
// Not the tail of an international number already counted
|
||||
if international.iter().any(|r| r.contains(&m.start())) {
|
||||
continue;
|
||||
}
|
||||
if (9..=11).contains(&number.len())
|
||||
&& stands_alone(text, m.start(), m.end())
|
||||
&& !text[..m.start()].ends_with('+')
|
||||
&& word_near(text, m.start(), m.end(), PHONE_WORDS)
|
||||
{
|
||||
findings.insert(number);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Date of birth --------------------------------------------------------
|
||||
|
||||
static DATE_ISO: LazyLock<Regex> = LazyLock::new(|| re(r"\b(\d{4})-(\d{2})-(\d{2})\b"));
|
||||
static DATE_NUMERIC: LazyLock<Regex> =
|
||||
LazyLock::new(|| re(r"\b(\d{1,2})[./-](\d{1,2})[./-](\d{4})\b"));
|
||||
static DATE_WORDS: LazyLock<Regex> = LazyLock::new(|| {
|
||||
re(
|
||||
r"(?i)\b(?:(\d{1,2})\s+(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*\.?,?\s+(\d{4})|(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*\.?\s+(\d{1,2}),?\s+(\d{4}))\b",
|
||||
)
|
||||
});
|
||||
|
||||
const BIRTH_WORDS: &[&str] = &[
|
||||
"born",
|
||||
"birth",
|
||||
"dob",
|
||||
"d.o.b",
|
||||
"birthday",
|
||||
"birthdate",
|
||||
"geburtsdatum",
|
||||
"geboren",
|
||||
"naissance",
|
||||
"né le",
|
||||
"née le",
|
||||
"nacimiento",
|
||||
"nacido",
|
||||
"nacida",
|
||||
"nascita",
|
||||
"nato il",
|
||||
"nata il",
|
||||
"geboortedatum",
|
||||
"födelsedatum",
|
||||
"fødselsdato",
|
||||
"syntymäaika",
|
||||
"urodzenia",
|
||||
"nascimento",
|
||||
];
|
||||
|
||||
fn month_number(name: &str) -> u32 {
|
||||
const MONTHS: [&str; 12] = [
|
||||
"jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec",
|
||||
];
|
||||
let name = name.to_lowercase();
|
||||
MONTHS
|
||||
.iter()
|
||||
.position(|m| *m == name)
|
||||
.map_or(0, |i| i as u32 + 1)
|
||||
}
|
||||
|
||||
fn date_of_birth(text: &str, findings: &mut Findings) {
|
||||
let mut add = |start: usize, end: usize, key: String| {
|
||||
if word_near(text, start, end, BIRTH_WORDS) {
|
||||
findings.insert(key);
|
||||
}
|
||||
};
|
||||
let num = |s: &str| s.parse::<u32>().unwrap_or(0);
|
||||
for c in DATE_ISO.captures_iter(text) {
|
||||
let (y, m, d) = (num(&c[1]), num(&c[2]), num(&c[3]));
|
||||
let whole = c.get(0).unwrap();
|
||||
if valid_date(y, m, d) {
|
||||
add(whole.start(), whole.end(), format!("{y:04}{m:02}{d:02}"));
|
||||
}
|
||||
}
|
||||
for c in DATE_NUMERIC.captures_iter(text) {
|
||||
let (a, b, y) = (num(&c[1]), num(&c[2]), num(&c[3]));
|
||||
let whole = c.get(0).unwrap();
|
||||
// Day first or month first: either reading that is a real date
|
||||
if valid_date(y, b, a) || valid_date(y, a, b) {
|
||||
add(whole.start(), whole.end(), whole.as_str().to_string());
|
||||
}
|
||||
}
|
||||
for c in DATE_WORDS.captures_iter(text) {
|
||||
let whole = c.get(0).unwrap();
|
||||
let (d, m, y) = match (c.get(1), c.get(4)) {
|
||||
(Some(d), _) => (num(d.as_str()), month_number(&c[2]), num(&c[3])),
|
||||
(_, Some(m)) => (num(&c[5]), month_number(m.as_str()), num(&c[6])),
|
||||
_ => continue,
|
||||
};
|
||||
if valid_date(y, m, d) {
|
||||
add(whole.start(), whole.end(), format!("{y:04}{m:02}{d:02}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Passport -------------------------------------------------------------
|
||||
|
||||
static PASSPORT: LazyLock<Regex> = LazyLock::new(|| re(r"\b[A-Z0-9]{6,9}\b"));
|
||||
|
||||
const PASSPORT_WORDS: &[&str] = &[
|
||||
"passport",
|
||||
"passeport",
|
||||
"reisepass",
|
||||
"pasaporte",
|
||||
"passaporto",
|
||||
"paspoort",
|
||||
"passnummer",
|
||||
"pass-nr",
|
||||
"passport no",
|
||||
"pasaporte n.º",
|
||||
"passaporte",
|
||||
];
|
||||
|
||||
fn passport(text: &str, findings: &mut Findings) {
|
||||
for m in PASSPORT.find_iter(text) {
|
||||
let value = m.as_str();
|
||||
if value.bytes().filter(u8::is_ascii_digit).count() >= 5
|
||||
&& word_near(text, m.start(), m.end(), PASSPORT_WORDS)
|
||||
{
|
||||
findings.insert(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Keys and credentials -------------------------------------------------
|
||||
|
||||
static PRIVATE_KEY: LazyLock<Regex> = LazyLock::new(|| {
|
||||
re(
|
||||
r"-----BEGIN (?:(?:RSA|EC|DSA|OPENSSH|ENCRYPTED|PGP) )?PRIVATE KEY(?: BLOCK)?-----\s*([A-Za-z0-9+/=:\s-]{0,64})",
|
||||
)
|
||||
});
|
||||
|
||||
fn private_key(text: &str, findings: &mut Findings) {
|
||||
for c in PRIVATE_KEY.captures_iter(text) {
|
||||
// Each key once, by the start of its body
|
||||
let body: String = c[1].chars().filter(|c| !c.is_whitespace()).collect();
|
||||
let whole = c.get(0).unwrap();
|
||||
findings.insert(if body.is_empty() {
|
||||
format!("@{}", whole.start())
|
||||
} else {
|
||||
body
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Published token formats: AWS access key IDs, GitHub tokens, Slack
|
||||
/// tokens, Stripe live secret and restricted keys, Google API keys.
|
||||
static CREDENTIAL: LazyLock<Regex> = LazyLock::new(|| {
|
||||
re(concat!(
|
||||
r"\b(?:",
|
||||
r"(?:AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}",
|
||||
r"|gh[pousr]_[A-Za-z0-9]{36}",
|
||||
r"|github_pat_[A-Za-z0-9_]{82}",
|
||||
r"|xox[abposr]-[A-Za-z0-9-]{10,72}",
|
||||
r"|(?:sk|rk)_live_[A-Za-z0-9]{24,99}",
|
||||
r"|AIza[0-9A-Za-z_-]{35}",
|
||||
r")\b"
|
||||
))
|
||||
});
|
||||
|
||||
fn credentials(text: &str, findings: &mut Findings) {
|
||||
for m in CREDENTIAL.find_iter(text) {
|
||||
findings.insert(m.as_str());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::mailflow::detectors::by_id;
|
||||
|
||||
fn count(id: &str, text: &str) -> usize {
|
||||
by_id(id).unwrap().count(text)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn payment_cards() {
|
||||
// Networks' and processors' published test numbers
|
||||
let text = "Visa 4242 4242 4242 4242, MC 5555-5555-5555-4444, Amex 378282246310005, \
|
||||
Discover 6011111111111117, JCB 3566002020360505, Diners 30569309025904, \
|
||||
UnionPay 6200000000000005, Mastercard 2-series 2223003122003222";
|
||||
assert_eq!(count("payment-card", text), 8);
|
||||
// Luhn fails, wrong network length, inside a longer number
|
||||
assert_eq!(count("payment-card", "4242424242424241"), 0);
|
||||
assert_eq!(count("payment-card", "378282246310005 0"), 1);
|
||||
assert_eq!(count("payment-card", "order 94242424242424242 shipped"), 0);
|
||||
// The same number twice counts once
|
||||
assert_eq!(
|
||||
count("payment-card", "4242424242424242 and 4242-4242-4242-4242"),
|
||||
1
|
||||
);
|
||||
// A card followed by a year
|
||||
assert_eq!(count("payment-card", "card 4242 4242 4242 4242 2031"), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ibans() {
|
||||
let text =
|
||||
"Pay GB29 NWBK 6016 1331 9268 19 or de89370400440532013000 (NL91ABNA0417164300).";
|
||||
assert_eq!(count("iban", text), 3);
|
||||
assert_eq!(count("iban", "GB29 NWBK 6016 1331 9268 18"), 0);
|
||||
// Runs into the next word: still found at the country's length
|
||||
assert_eq!(count("iban", "IBAN NL91ABNA0417164300 BIC ABNANL2A"), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn swift_codes_need_a_word() {
|
||||
assert_eq!(count("swift-bic", "SWIFT: DEUTDEFF500"), 1);
|
||||
assert_eq!(count("swift-bic", "BIC NWBKGB2L"), 1);
|
||||
assert_eq!(count("swift-bic", "HAPPYDAYS DEUTDEFF"), 0);
|
||||
// Not a country in positions 5–6
|
||||
assert_eq!(count("swift-bic", "BIC DEUTZZFF"), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn email_and_phone_lists() {
|
||||
let list = "[email protected], [email protected], [email protected], [email protected]";
|
||||
assert_eq!(count("email-addresses", list), 3);
|
||||
assert_eq!(
|
||||
count("phone-numbers", "+44 20 7946 0958, +1 (415) 555-2671"),
|
||||
2
|
||||
);
|
||||
assert_eq!(count("phone-numbers", "call 020 7946 0958"), 0);
|
||||
assert_eq!(count("phone-numbers", "Tel: 020 7946 0958"), 1);
|
||||
assert_eq!(count("phone-numbers", "invoice 020 7946 0958"), 0);
|
||||
// One number, not also its national tail
|
||||
assert_eq!(count("phone-numbers", "Tel: +44 20 7946 0958"), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dates_of_birth() {
|
||||
assert_eq!(count("date-of-birth", "DOB: 1984-02-29"), 1);
|
||||
assert_eq!(count("date-of-birth", "Geburtsdatum 31.12.1970"), 1);
|
||||
assert_eq!(count("date-of-birth", "born on March 3, 1962"), 1);
|
||||
assert_eq!(count("date-of-birth", "date of birth 3 Mar 1962"), 1);
|
||||
// Not a real date, no word, a meeting
|
||||
assert_eq!(count("date-of-birth", "DOB: 1985-02-29"), 0);
|
||||
assert_eq!(count("date-of-birth", "invoice 1984-02-29"), 0);
|
||||
assert_eq!(count("date-of-birth", "Meeting on 12/05/2026"), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn passports_need_a_word() {
|
||||
assert_eq!(count("passport", "Passport number: 533380006"), 1);
|
||||
assert_eq!(count("passport", "Reisepass C01X00T47"), 1);
|
||||
assert_eq!(count("passport", "Order 533380006 shipped"), 0);
|
||||
// Mostly letters: a word, not a number
|
||||
assert_eq!(count("passport", "passport PASSWORD"), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keys_and_credentials() {
|
||||
let key = "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQ\n-----END OPENSSH PRIVATE KEY-----";
|
||||
assert_eq!(count("private-key", key), 1);
|
||||
assert_eq!(count("private-key", "-----BEGIN PUBLIC KEY-----\nMFkw"), 0);
|
||||
// Documentation examples of each format
|
||||
let tokens = "AKIAIOSFODNN7EXAMPLE ghp_0123456789abcdefghijklmnopqrstuvwxyz \
|
||||
AIzaSyA-0123456789abcdefghijklmnopqrstu";
|
||||
assert_eq!(count("credentials", tokens), 3);
|
||||
assert_eq!(count("credentials", "AKIA123 ghp_short"), 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! Asian identifiers (§2.3): India's Aadhaar and PAN, China's resident ID,
|
||||
//! Japan's My Number, Singapore's NRIC and FIN, and South Korea's resident
|
||||
//! registration number.
|
||||
|
||||
use super::{
|
||||
Detector, Findings, Region, Strength, digit_values, valid_date, valid_short_date, word_near,
|
||||
};
|
||||
use regex::Regex;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub static DETECTORS: &[Detector] = &[
|
||||
Detector::new(
|
||||
"in-aadhaar",
|
||||
"India: Aadhaar",
|
||||
Region::Asia,
|
||||
Strength::Checked,
|
||||
in_aadhaar,
|
||||
),
|
||||
Detector::new(
|
||||
"in-pan",
|
||||
"India: PAN",
|
||||
Region::Asia,
|
||||
Strength::NeedsWord,
|
||||
in_pan,
|
||||
),
|
||||
Detector::new(
|
||||
"cn-resident-id",
|
||||
"China: resident ID",
|
||||
Region::Asia,
|
||||
Strength::Checked,
|
||||
cn_resident_id,
|
||||
),
|
||||
Detector::new(
|
||||
"jp-my-number",
|
||||
"Japan: My Number",
|
||||
Region::Asia,
|
||||
Strength::Checked,
|
||||
jp_my_number,
|
||||
),
|
||||
Detector::new(
|
||||
"sg-nric",
|
||||
"Singapore: NRIC and FIN",
|
||||
Region::Asia,
|
||||
Strength::Checked,
|
||||
sg_nric,
|
||||
),
|
||||
Detector::new(
|
||||
"kr-rrn",
|
||||
"South Korea: resident registration number",
|
||||
Region::Asia,
|
||||
Strength::NeedsWord,
|
||||
kr_rrn,
|
||||
),
|
||||
];
|
||||
|
||||
fn re(pattern: &str) -> Regex {
|
||||
Regex::new(pattern).expect("detector pattern")
|
||||
}
|
||||
|
||||
/// Twelve digits written in fours, or bare.
|
||||
static TWELVE: LazyLock<Regex> = LazyLock::new(|| re(r"\b(\d{4})( ?)(\d{4})( ?)(\d{4})\b"));
|
||||
|
||||
const VERHOEFF_D: [[u8; 10]; 10] = [
|
||||
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
|
||||
[1, 2, 3, 4, 0, 6, 7, 8, 9, 5],
|
||||
[2, 3, 4, 0, 1, 7, 8, 9, 5, 6],
|
||||
[3, 4, 0, 1, 2, 8, 9, 5, 6, 7],
|
||||
[4, 0, 1, 2, 3, 9, 5, 6, 7, 8],
|
||||
[5, 9, 8, 7, 6, 0, 4, 3, 2, 1],
|
||||
[6, 5, 9, 8, 7, 1, 0, 4, 3, 2],
|
||||
[7, 6, 5, 9, 8, 2, 1, 0, 4, 3],
|
||||
[8, 7, 6, 5, 9, 3, 2, 1, 0, 4],
|
||||
[9, 8, 7, 6, 5, 4, 3, 2, 1, 0],
|
||||
];
|
||||
const VERHOEFF_P: [[u8; 10]; 8] = [
|
||||
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
|
||||
[1, 5, 7, 6, 2, 8, 3, 0, 9, 4],
|
||||
[5, 8, 0, 3, 7, 9, 6, 1, 4, 2],
|
||||
[8, 9, 1, 6, 0, 4, 3, 5, 2, 7],
|
||||
[9, 4, 5, 3, 1, 2, 6, 8, 7, 0],
|
||||
[4, 2, 8, 6, 5, 7, 3, 9, 0, 1],
|
||||
[2, 7, 9, 3, 8, 0, 6, 4, 1, 5],
|
||||
[7, 0, 4, 6, 9, 1, 3, 2, 5, 8],
|
||||
];
|
||||
|
||||
/// The Verhoeff check (dihedral group D5).
|
||||
pub fn verhoeff(n: &str) -> bool {
|
||||
let mut c = 0u8;
|
||||
for (i, b) in n.bytes().rev().enumerate() {
|
||||
c = VERHOEFF_D[c as usize][VERHOEFF_P[i % 8][(b - b'0') as usize] as usize];
|
||||
}
|
||||
c == 0
|
||||
}
|
||||
|
||||
const AADHAAR_WORDS: &[&str] = &["aadhaar", "aadhar", "uidai", "uid"];
|
||||
|
||||
fn in_aadhaar(text: &str, findings: &mut Findings) {
|
||||
for c in TWELVE.captures_iter(text) {
|
||||
let whole = c.get(0).unwrap();
|
||||
let n = format!("{}{}{}", &c[1], &c[3], &c[5]);
|
||||
let written = &c[2] == " " && &c[4] == " ";
|
||||
// Never starts with 0 or 1
|
||||
if !n.starts_with(['0', '1'])
|
||||
&& verhoeff(&n)
|
||||
&& (written || word_near(text, whole.start(), whole.end(), AADHAAR_WORDS))
|
||||
{
|
||||
findings.insert(n);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Five letters (the fourth names the holder's type), four digits, a letter.
|
||||
static PAN: LazyLock<Regex> = LazyLock::new(|| re(r"\b[A-Z]{3}[ABCFGHLJPTK][A-Z]\d{4}[A-Z]\b"));
|
||||
|
||||
const PAN_WORDS: &[&str] = &["pan", "pan card", "permanent account number", "income tax"];
|
||||
|
||||
fn in_pan(text: &str, findings: &mut Findings) {
|
||||
for m in PAN.find_iter(text) {
|
||||
if word_near(text, m.start(), m.end(), PAN_WORDS) {
|
||||
findings.insert(m.as_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Region, birth date `YYYYMMDD`, sequence, then the ISO 7064 MOD 11-2
|
||||
/// check (0–9 or X).
|
||||
static CN_ID: LazyLock<Regex> =
|
||||
LazyLock::new(|| re(r"(?i)\b[1-8]\d{5}(\d{4})(\d{2})(\d{2})\d{3}[\dX]\b"));
|
||||
|
||||
pub fn cn_id_valid(id: &str) -> bool {
|
||||
const WEIGHTS: [u32; 17] = [7, 9, 10, 5, 8, 4, 2, 1, 6, 3, 7, 9, 10, 5, 8, 4, 2];
|
||||
const CHECKS: &[u8] = b"10X98765432";
|
||||
let sum: u32 = digit_values(&id[..17])
|
||||
.iter()
|
||||
.zip(WEIGHTS)
|
||||
.map(|(a, w)| a * w)
|
||||
.sum();
|
||||
CHECKS[(sum % 11) as usize] == id.as_bytes()[17].to_ascii_uppercase()
|
||||
}
|
||||
|
||||
fn cn_resident_id(text: &str, findings: &mut Findings) {
|
||||
for c in CN_ID.captures_iter(text) {
|
||||
let id = c[0].to_ascii_uppercase();
|
||||
let (y, m, d) = (num(&c[1]), num(&c[2]), num(&c[3]));
|
||||
if valid_date(y, m, d) && cn_id_valid(&id) {
|
||||
findings.insert(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// My Number: weights 2–7 then 2–6 from the right; a remainder of 0 or 1
|
||||
/// gives 0, else 11 minus it.
|
||||
pub fn my_number_valid(n: &str) -> bool {
|
||||
let d = digit_values(n);
|
||||
let sum: u32 = (1..=11)
|
||||
.map(|i| d[11 - i] * if i <= 6 { i as u32 + 1 } else { i as u32 - 5 })
|
||||
.sum();
|
||||
let check = match sum % 11 {
|
||||
0 | 1 => 0,
|
||||
r => 11 - r,
|
||||
};
|
||||
check == d[11]
|
||||
}
|
||||
|
||||
const MY_NUMBER_WORDS: &[&str] = &[
|
||||
"my number",
|
||||
"mynumber",
|
||||
"マイナンバー",
|
||||
"個人番号",
|
||||
"kojin bango",
|
||||
];
|
||||
|
||||
fn jp_my_number(text: &str, findings: &mut Findings) {
|
||||
for c in TWELVE.captures_iter(text) {
|
||||
let whole = c.get(0).unwrap();
|
||||
let n = format!("{}{}{}", &c[1], &c[3], &c[5]);
|
||||
let written = &c[2] == " " && &c[4] == " ";
|
||||
if my_number_valid(&n)
|
||||
&& (written || word_near(text, whole.start(), whole.end(), MY_NUMBER_WORDS))
|
||||
{
|
||||
findings.insert(n);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static NRIC: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\b([STFGM])(\d{7})([A-Z])\b"));
|
||||
|
||||
/// Weights 2, 7, 6, 5, 4, 3, 2; T and G add 4, M adds 3; each series has its
|
||||
/// own table of check letters.
|
||||
fn nric_valid(prefix: u8, digits: &str, check: u8) -> bool {
|
||||
let sum: u32 = digit_values(digits)
|
||||
.iter()
|
||||
.zip([2, 7, 6, 5, 4, 3, 2])
|
||||
.map(|(a, w)| a * w)
|
||||
.sum::<u32>()
|
||||
+ match prefix {
|
||||
b'T' | b'G' => 4,
|
||||
b'M' => 3,
|
||||
_ => 0,
|
||||
};
|
||||
let table: &[u8] = match prefix {
|
||||
b'S' | b'T' => b"JZIHGFEDCBA",
|
||||
b'F' | b'G' => b"XWUTRQPNMLK",
|
||||
_ => b"KLJNPQRTUWX",
|
||||
};
|
||||
table[(sum % 11) as usize] == check
|
||||
}
|
||||
|
||||
fn sg_nric(text: &str, findings: &mut Findings) {
|
||||
for c in NRIC.captures_iter(text) {
|
||||
let id = c[0].to_ascii_uppercase();
|
||||
let bytes = id.as_bytes();
|
||||
if nric_valid(bytes[0], &c[2], bytes[8]) {
|
||||
findings.insert(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `YYMMDD-GNNNNNN`, the seventh digit giving sex and century.
|
||||
static RRN: LazyLock<Regex> = LazyLock::new(|| re(r"\b(\d{2})(\d{2})(\d{2})-?([1-8])\d{6}\b"));
|
||||
|
||||
const RRN_WORDS: &[&str] = &["주민등록번호", "주민번호", "resident registration", "rrn"];
|
||||
|
||||
fn kr_rrn(text: &str, findings: &mut Findings) {
|
||||
for c in RRN.captures_iter(text) {
|
||||
let whole = c.get(0).unwrap();
|
||||
if valid_short_date(num(&c[1]), num(&c[2]), num(&c[3]))
|
||||
&& word_near(text, whole.start(), whole.end(), RRN_WORDS)
|
||||
{
|
||||
findings.insert(whole.as_str().replace('-', ""));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn num(s: &str) -> u32 {
|
||||
s.parse().unwrap_or(0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::mailflow::detectors::by_id;
|
||||
|
||||
fn count(id: &str, text: &str) -> usize {
|
||||
by_id(id).unwrap().count(text)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn india() {
|
||||
assert_eq!(count("in-aadhaar", "2345 6789 0124"), 1);
|
||||
assert_eq!(count("in-aadhaar", "2345 6789 0125"), 0);
|
||||
assert_eq!(count("in-aadhaar", "order 234567890124"), 0);
|
||||
assert_eq!(count("in-aadhaar", "Aadhaar 234567890124"), 1);
|
||||
assert_eq!(count("in-pan", "PAN: ABCPE1234F"), 1);
|
||||
assert_eq!(count("in-pan", "ABCPE1234F"), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn china_japan() {
|
||||
assert_eq!(count("cn-resident-id", "11010519491231002X"), 1);
|
||||
assert_eq!(count("cn-resident-id", "110105194912310021"), 0);
|
||||
assert_eq!(count("cn-resident-id", "11010519491331002X"), 0);
|
||||
assert_eq!(count("jp-my-number", "1234 5678 9018"), 1);
|
||||
assert_eq!(count("jp-my-number", "1234 5678 9017"), 0);
|
||||
assert_eq!(count("jp-my-number", "マイナンバー 123456789018"), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn singapore_korea() {
|
||||
assert_eq!(count("sg-nric", "S1234567D and T1234567J"), 2);
|
||||
assert_eq!(count("sg-nric", "S1234567E"), 0);
|
||||
assert_eq!(count("kr-rrn", "주민등록번호 800101-1234567"), 1);
|
||||
assert_eq!(count("kr-rrn", "800101-1234567"), 0);
|
||||
assert_eq!(count("kr-rrn", "RRN 801301-1234567"), 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! Australian identifiers (§2.3): the ATO's Tax File Number and the Medicare
|
||||
//! card number.
|
||||
|
||||
use super::{Detector, Findings, Region, Strength, word_near};
|
||||
use regex::Regex;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub static DETECTORS: &[Detector] = &[
|
||||
Detector::new(
|
||||
"au-tfn",
|
||||
"Australian Tax File Number",
|
||||
Region::Australia,
|
||||
Strength::Checked,
|
||||
tfn,
|
||||
),
|
||||
Detector::new(
|
||||
"au-medicare",
|
||||
"Australian Medicare number",
|
||||
Region::Australia,
|
||||
Strength::Checked,
|
||||
medicare,
|
||||
),
|
||||
];
|
||||
|
||||
fn re(pattern: &str) -> Regex {
|
||||
Regex::new(pattern).expect("detector pattern")
|
||||
}
|
||||
|
||||
/// `NNN NNN NNN` stands alone; bare digits (eight or nine) need a word.
|
||||
static TFN: LazyLock<Regex> = LazyLock::new(|| re(r"\b(\d{3})( ?)(\d{3})( ?)(\d{2,3})\b"));
|
||||
|
||||
/// Weighted sum mod 11, with the ATO's weights for 9- and 8-digit numbers.
|
||||
pub fn tfn_valid(n: &str) -> bool {
|
||||
let weights: &[u32] = match n.len() {
|
||||
9 => &[1, 4, 3, 7, 5, 8, 6, 9, 10],
|
||||
8 => &[10, 7, 8, 4, 6, 3, 5, 1],
|
||||
_ => return false,
|
||||
};
|
||||
n.bytes()
|
||||
.zip(weights)
|
||||
.map(|(b, w)| u32::from(b - b'0') * w)
|
||||
.sum::<u32>()
|
||||
% 11
|
||||
== 0
|
||||
}
|
||||
|
||||
const TFN_WORDS: &[&str] = &["tfn", "tax file number", "tax file no"];
|
||||
|
||||
fn tfn(text: &str, findings: &mut Findings) {
|
||||
for c in TFN.captures_iter(text) {
|
||||
let whole = c.get(0).unwrap();
|
||||
let n = format!("{}{}{}", &c[1], &c[3], &c[5]);
|
||||
let written = n.len() == 9 && c[2] == *" " && c[4] == *" ";
|
||||
if tfn_valid(&n) && (written || word_near(text, whole.start(), whole.end(), TFN_WORDS)) {
|
||||
findings.insert(n);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `NNNN NNNNN N` (and an optional issue number) stands alone; bare digits
|
||||
/// need a word.
|
||||
static MEDICARE: LazyLock<Regex> =
|
||||
LazyLock::new(|| re(r"\b([2-6]\d{3})( ?)(\d{5})( ?)(\d)(?:[ -]?\d)?\b"));
|
||||
|
||||
/// The ninth digit is the weighted sum (1, 3, 7, 9, 1, 3, 7, 9) of the first
|
||||
/// eight, mod 10.
|
||||
pub fn medicare_valid(n: &str) -> bool {
|
||||
let d: Vec<u32> = n.bytes().map(|b| u32::from(b - b'0')).collect();
|
||||
d.len() >= 9
|
||||
&& d[..8]
|
||||
.iter()
|
||||
.zip([1, 3, 7, 9, 1, 3, 7, 9])
|
||||
.map(|(a, w)| a * w)
|
||||
.sum::<u32>()
|
||||
% 10
|
||||
== d[8]
|
||||
}
|
||||
|
||||
const MEDICARE_WORDS: &[&str] = &[
|
||||
"medicare",
|
||||
"medicare card",
|
||||
"medicare no",
|
||||
"medicare number",
|
||||
];
|
||||
|
||||
fn medicare(text: &str, findings: &mut Findings) {
|
||||
for c in MEDICARE.captures_iter(text) {
|
||||
let whole = c.get(0).unwrap();
|
||||
let n = format!("{}{}{}", &c[1], &c[3], &c[5]);
|
||||
let written = c[2] == *" " && c[4] == *" ";
|
||||
if medicare_valid(&n)
|
||||
&& (written || word_near(text, whole.start(), whole.end(), MEDICARE_WORDS))
|
||||
{
|
||||
findings.insert(n);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::mailflow::detectors::by_id;
|
||||
|
||||
fn count(id: &str, text: &str) -> usize {
|
||||
by_id(id).unwrap().count(text)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tax_file_numbers() {
|
||||
assert_eq!(count("au-tfn", "TFN 123 456 782"), 1);
|
||||
assert_eq!(count("au-tfn", "123 456 789"), 0);
|
||||
assert_eq!(count("au-tfn", "order 123456782"), 0);
|
||||
assert_eq!(count("au-tfn", "tax file number 123456782"), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn medicare_numbers() {
|
||||
assert_eq!(count("au-medicare", "2123 45670 1"), 1);
|
||||
assert_eq!(count("au-medicare", "2123 45671 1"), 0);
|
||||
assert_eq!(count("au-medicare", "ref 2123456701"), 0);
|
||||
assert_eq!(count("au-medicare", "Medicare 2123456701"), 1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! Canadian identifiers (§2.3): the Social Insurance Number.
|
||||
|
||||
use super::{Detector, Findings, Region, Strength, checks, word_near};
|
||||
use regex::Regex;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub static DETECTORS: &[Detector] = &[Detector::new(
|
||||
"ca-sin",
|
||||
"Canadian Social Insurance Number",
|
||||
Region::Canada,
|
||||
Strength::Checked,
|
||||
sin,
|
||||
)];
|
||||
|
||||
/// `NNN NNN NNN` or `NNN-NNN-NNN` stands alone; nine bare digits need a word.
|
||||
static SIN: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r"\b(\d{3})([ -]?)(\d{3})([ -]?)(\d{3})\b").expect("detector pattern")
|
||||
});
|
||||
|
||||
const SIN_WORDS: &[&str] = &[
|
||||
"sin",
|
||||
"social insurance",
|
||||
"nas",
|
||||
"numéro d'assurance sociale",
|
||||
"assurance sociale",
|
||||
];
|
||||
|
||||
fn sin(text: &str, findings: &mut Findings) {
|
||||
for c in SIN.captures_iter(text) {
|
||||
let whole = c.get(0).unwrap();
|
||||
let n = format!("{}{}{}", &c[1], &c[3], &c[5]);
|
||||
let written = !c[2].is_empty() && c[2] == c[4];
|
||||
// 0 and 8 are never issued as a first digit
|
||||
if !n.starts_with(['0', '8'])
|
||||
&& checks::luhn(&n)
|
||||
&& (written || word_near(text, whole.start(), whole.end(), SIN_WORDS))
|
||||
{
|
||||
findings.insert(n);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::mailflow::detectors::by_id;
|
||||
|
||||
fn count(text: &str) -> usize {
|
||||
by_id("ca-sin").unwrap().count(text)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn social_insurance_numbers() {
|
||||
assert_eq!(count("130 692 544 and 193-456-787"), 2);
|
||||
assert_eq!(count("130 692 545"), 0);
|
||||
// The government's printed example starts with 0, never issued
|
||||
assert_eq!(count("046 454 286"), 0);
|
||||
assert_eq!(count("order 130692544"), 0);
|
||||
assert_eq!(count("SIN: 130692544"), 1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! Check-digit algorithms, each from its public definition.
|
||||
|
||||
/// The Luhn check (ISO/IEC 7812-1, Annex B) over a string of ASCII digits.
|
||||
pub fn luhn(digits: &str) -> bool {
|
||||
if digits.len() < 2 || !digits.bytes().all(|b| b.is_ascii_digit()) {
|
||||
return false;
|
||||
}
|
||||
let sum: u32 = digits
|
||||
.bytes()
|
||||
.rev()
|
||||
.enumerate()
|
||||
.map(|(i, b)| {
|
||||
let d = u32::from(b - b'0');
|
||||
if i % 2 == 1 {
|
||||
let d = d * 2;
|
||||
if d > 9 { d - 9 } else { d }
|
||||
} else {
|
||||
d
|
||||
}
|
||||
})
|
||||
.sum();
|
||||
sum.is_multiple_of(10)
|
||||
}
|
||||
|
||||
/// ISO 13616 IBAN lengths, by country, from the IBAN registry.
|
||||
const IBAN_LENGTHS: &[(&str, usize)] = &[
|
||||
("AD", 24),
|
||||
("AE", 23),
|
||||
("AL", 28),
|
||||
("AT", 20),
|
||||
("AZ", 28),
|
||||
("BA", 20),
|
||||
("BE", 16),
|
||||
("BG", 22),
|
||||
("BH", 22),
|
||||
("BI", 27),
|
||||
("BR", 29),
|
||||
("BY", 28),
|
||||
("CH", 21),
|
||||
("CR", 22),
|
||||
("CY", 28),
|
||||
("CZ", 24),
|
||||
("DE", 22),
|
||||
("DJ", 27),
|
||||
("DK", 18),
|
||||
("DO", 28),
|
||||
("EE", 20),
|
||||
("EG", 29),
|
||||
("ES", 24),
|
||||
("FI", 18),
|
||||
("FK", 18),
|
||||
("FO", 18),
|
||||
("FR", 27),
|
||||
("GB", 22),
|
||||
("GE", 22),
|
||||
("GI", 23),
|
||||
("GL", 18),
|
||||
("GR", 27),
|
||||
("GT", 28),
|
||||
("HN", 28),
|
||||
("HR", 21),
|
||||
("HU", 28),
|
||||
("IE", 22),
|
||||
("IL", 23),
|
||||
("IQ", 23),
|
||||
("IS", 26),
|
||||
("IT", 27),
|
||||
("JO", 30),
|
||||
("KW", 30),
|
||||
("KZ", 20),
|
||||
("LB", 28),
|
||||
("LC", 32),
|
||||
("LI", 21),
|
||||
("LT", 20),
|
||||
("LU", 20),
|
||||
("LV", 21),
|
||||
("LY", 25),
|
||||
("MC", 27),
|
||||
("MD", 24),
|
||||
("ME", 22),
|
||||
("MK", 19),
|
||||
("MN", 20),
|
||||
("MR", 27),
|
||||
("MT", 31),
|
||||
("MU", 30),
|
||||
("NI", 28),
|
||||
("NL", 18),
|
||||
("NO", 15),
|
||||
("OM", 23),
|
||||
("PK", 24),
|
||||
("PL", 28),
|
||||
("PS", 29),
|
||||
("PT", 25),
|
||||
("QA", 29),
|
||||
("RO", 24),
|
||||
("RS", 22),
|
||||
("RU", 33),
|
||||
("SA", 24),
|
||||
("SC", 31),
|
||||
("SD", 18),
|
||||
("SE", 24),
|
||||
("SI", 19),
|
||||
("SK", 24),
|
||||
("SM", 27),
|
||||
("SO", 23),
|
||||
("ST", 25),
|
||||
("SV", 28),
|
||||
("TL", 23),
|
||||
("TN", 24),
|
||||
("TR", 26),
|
||||
("UA", 29),
|
||||
("VA", 22),
|
||||
("VG", 24),
|
||||
("XK", 20),
|
||||
("YE", 30),
|
||||
];
|
||||
|
||||
/// The IBAN length for a country code, if the country uses IBANs.
|
||||
pub fn iban_length(country: &str) -> Option<usize> {
|
||||
IBAN_LENGTHS
|
||||
.iter()
|
||||
.find(|(code, _)| *code == country)
|
||||
.map(|(_, len)| *len)
|
||||
}
|
||||
|
||||
/// ISO 13616 / ISO 7064 MOD 97-10 over an IBAN with no spaces, upper case:
|
||||
/// move the first four characters to the end, turn letters into 10–35, and
|
||||
/// the number mod 97 must be 1. Also checks the country's length.
|
||||
pub fn iban(iban: &str) -> bool {
|
||||
if iban.len() < 5
|
||||
|| !iban
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_uppercase() || b.is_ascii_digit())
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if iban_length(&iban[..2]) != Some(iban.len())
|
||||
|| !iban[2..4].bytes().all(|b| b.is_ascii_digit())
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let mut remainder: u32 = 0;
|
||||
for b in iban[4..].bytes().chain(iban[..4].bytes()) {
|
||||
let value = if b.is_ascii_digit() {
|
||||
u32::from(b - b'0')
|
||||
} else {
|
||||
u32::from(b - b'A') + 10
|
||||
};
|
||||
remainder = if value >= 10 {
|
||||
(remainder * 100 + value) % 97
|
||||
} else {
|
||||
(remainder * 10 + value) % 97
|
||||
};
|
||||
}
|
||||
remainder == 1
|
||||
}
|
||||
|
||||
/// ISO 3166-1 alpha-2 country codes, for SWIFT/BIC positions 5–6.
|
||||
const COUNTRIES: &str = "AD AE AF AG AI AL AM AO AQ AR AS AT AU AW AX AZ BA BB BD BE BF BG BH BI BJ \
|
||||
BL BM BN BO BQ BR BS BT BV BW BY BZ CA CC CD CF CG CH CI CK CL CM CN CO CR CU CV CW CX CY CZ DE DJ \
|
||||
DK DM DO DZ EC EE EG EH ER ES ET FI FJ FK FM FO FR GA GB GD GE GF GG GH GI GL GM GN GP GQ GR GS GT \
|
||||
GU GW GY HK HM HN HR HT HU ID IE IL IM IN IO IQ IR IS IT JE JM JO JP KE KG KH KI KM KN KP KR KW KY \
|
||||
KZ LA LB LC LI LK LR LS LT LU LV LY MA MC MD ME MF MG MH MK ML MM MN MO MP MQ MR MS MT MU MV MW MX \
|
||||
MY MZ NA NC NE NF NG NI NL NO NP NR NU NZ OM PA PE PF PG PH PK PL PM PN PR PS PT PW PY QA RE RO RS \
|
||||
RU RW SA SB SC SD SE SG SH SI SJ SK SL SM SN SO SR SS ST SV SX SY SZ TC TD TF TG TH TJ TK TL TM TN \
|
||||
TO TR TT TV TW TZ UA UG UM US UY UZ VA VC VE VG VI VN VU WF WS XK YE YT ZA ZM ZW";
|
||||
|
||||
pub fn is_country(code: &str) -> bool {
|
||||
code.len() == 2 && COUNTRIES.split(' ').any(|c| c == code)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn luhn_known_numbers() {
|
||||
// Published test card numbers
|
||||
for good in [
|
||||
"4242424242424242",
|
||||
"5555555555554444",
|
||||
"378282246310005",
|
||||
"79927398713",
|
||||
] {
|
||||
assert!(luhn(good), "{good}");
|
||||
}
|
||||
for bad in ["4242424242424241", "79927398710", "1", "12a4"] {
|
||||
assert!(!luhn(bad), "{bad}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn iban_registry_examples() {
|
||||
// The IBAN registry's own examples
|
||||
for good in [
|
||||
"GB29NWBK60161331926819",
|
||||
"DE89370400440532013000",
|
||||
"FR1420041010050500013M02606",
|
||||
"NL91ABNA0417164300",
|
||||
"BE68539007547034",
|
||||
"NO9386011117947",
|
||||
"CH9300762011623852957",
|
||||
] {
|
||||
assert!(iban(good), "{good}");
|
||||
}
|
||||
for bad in [
|
||||
"GB29NWBK60161331926818", // check fails
|
||||
"GB29NWBK6016133192681", // too short for GB
|
||||
"ZZ29NWBK60161331926819", // no such country
|
||||
"DE8937040044053201300A", // letters where DE has none still fail mod 97
|
||||
] {
|
||||
assert!(!iban(bad), "{bad}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn countries() {
|
||||
assert!(is_country("DE") && is_country("US") && is_country("XK"));
|
||||
assert!(!is_country("ZZ") && !is_country("D"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,646 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! European Union national identifiers (§2.3), each from its issuer's
|
||||
//! published rules. An identifier that is only digits and whose check a
|
||||
//! random number passes often (mod 10, mod 11) counts alone only in its
|
||||
//! written form, and as bare digits only beside a word.
|
||||
|
||||
use super::{
|
||||
Detector, Findings, Region, Strength, checks, digit_values, stands_alone, valid_short_date,
|
||||
word_near,
|
||||
};
|
||||
use regex::Regex;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub static DETECTORS: &[Detector] = &[
|
||||
Detector::new(
|
||||
"de-tax-id",
|
||||
"Germany: tax ID (Steuer-ID)",
|
||||
Region::Eu,
|
||||
Strength::Checked,
|
||||
de_tax_id,
|
||||
),
|
||||
Detector::new(
|
||||
"de-id-card",
|
||||
"Germany: ID card number",
|
||||
Region::Eu,
|
||||
Strength::Checked,
|
||||
de_id_card,
|
||||
),
|
||||
Detector::new(
|
||||
"fr-nir",
|
||||
"France: social security number (NIR)",
|
||||
Region::Eu,
|
||||
Strength::Checked,
|
||||
fr_nir,
|
||||
),
|
||||
Detector::new(
|
||||
"es-dni-nie",
|
||||
"Spain: DNI and NIE",
|
||||
Region::Eu,
|
||||
Strength::Checked,
|
||||
es_dni_nie,
|
||||
),
|
||||
Detector::new(
|
||||
"it-codice-fiscale",
|
||||
"Italy: codice fiscale",
|
||||
Region::Eu,
|
||||
Strength::Checked,
|
||||
it_codice_fiscale,
|
||||
),
|
||||
Detector::new(
|
||||
"nl-bsn",
|
||||
"Netherlands: BSN",
|
||||
Region::Eu,
|
||||
Strength::Checked,
|
||||
nl_bsn,
|
||||
),
|
||||
Detector::new(
|
||||
"be-national-number",
|
||||
"Belgium: national number",
|
||||
Region::Eu,
|
||||
Strength::Checked,
|
||||
be_national_number,
|
||||
),
|
||||
Detector::new(
|
||||
"pl-pesel",
|
||||
"Poland: PESEL",
|
||||
Region::Eu,
|
||||
Strength::Checked,
|
||||
pl_pesel,
|
||||
),
|
||||
Detector::new(
|
||||
"se-personnummer",
|
||||
"Sweden: personnummer",
|
||||
Region::Eu,
|
||||
Strength::Checked,
|
||||
se_personnummer,
|
||||
),
|
||||
Detector::new(
|
||||
"dk-cpr",
|
||||
"Denmark: CPR number",
|
||||
Region::Eu,
|
||||
Strength::NeedsWord,
|
||||
dk_cpr,
|
||||
),
|
||||
Detector::new(
|
||||
"fi-hetu",
|
||||
"Finland: personal identity code",
|
||||
Region::Eu,
|
||||
Strength::Checked,
|
||||
fi_hetu,
|
||||
),
|
||||
Detector::new(
|
||||
"ie-pps",
|
||||
"Ireland: PPS number",
|
||||
Region::Eu,
|
||||
Strength::Checked,
|
||||
ie_pps,
|
||||
),
|
||||
Detector::new(
|
||||
"pt-nif",
|
||||
"Portugal: NIF",
|
||||
Region::Eu,
|
||||
Strength::Checked,
|
||||
pt_nif,
|
||||
),
|
||||
Detector::new(
|
||||
"at-svnr",
|
||||
"Austria: social insurance number",
|
||||
Region::Eu,
|
||||
Strength::Checked,
|
||||
at_svnr,
|
||||
),
|
||||
];
|
||||
|
||||
fn re(pattern: &str) -> Regex {
|
||||
Regex::new(pattern).expect("detector pattern")
|
||||
}
|
||||
|
||||
fn num(s: &str) -> u32 {
|
||||
s.parse().unwrap_or(0)
|
||||
}
|
||||
|
||||
// --- Germany --------------------------------------------------------------
|
||||
|
||||
/// Eleven digits, written `86 095 742 719` on the BZSt's letters.
|
||||
static DE_TAX: LazyLock<Regex> = LazyLock::new(|| re(r"\b\d{2}( ?)\d{3}( ?)\d{3}( ?)\d{3}\b"));
|
||||
|
||||
/// ISO 7064 MOD 11,10; no leading zero; in the first ten digits one digit
|
||||
/// appears two or three times and every other at most once.
|
||||
pub fn de_tax_id_valid(n: &str) -> bool {
|
||||
let d = digit_values(n);
|
||||
if d.len() != 11 || d[0] == 0 {
|
||||
return false;
|
||||
}
|
||||
let mut counts = [0u8; 10];
|
||||
for &x in &d[..10] {
|
||||
counts[x as usize] += 1;
|
||||
}
|
||||
let repeated = counts.iter().filter(|&&c| c >= 2).count();
|
||||
if repeated != 1 || counts.iter().any(|&c| c > 3) {
|
||||
return false;
|
||||
}
|
||||
let mut product = 10;
|
||||
for &x in &d[..10] {
|
||||
let mut sum = (x + product) % 10;
|
||||
if sum == 0 {
|
||||
sum = 10;
|
||||
}
|
||||
product = (2 * sum) % 11;
|
||||
}
|
||||
let check = match 11 - product {
|
||||
10 => 0,
|
||||
c => c,
|
||||
};
|
||||
check == d[10]
|
||||
}
|
||||
|
||||
const DE_TAX_WORDS: &[&str] = &[
|
||||
"steuer-id",
|
||||
"steueridentifikationsnummer",
|
||||
"steuerliche identifikationsnummer",
|
||||
"idnr",
|
||||
"identifikationsnummer",
|
||||
"tax id",
|
||||
];
|
||||
|
||||
fn de_tax_id(text: &str, findings: &mut Findings) {
|
||||
for c in DE_TAX.captures_iter(text) {
|
||||
let whole = c.get(0).unwrap();
|
||||
let written = [&c[1], &c[2], &c[3]].iter().all(|s| *s == " ");
|
||||
let n: String = whole.as_str().replace(' ', "");
|
||||
if de_tax_id_valid(&n)
|
||||
&& (written || word_near(text, whole.start(), whole.end(), DE_TAX_WORDS))
|
||||
{
|
||||
findings.insert(n);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The ID card's document number: a letter from the card's alphabet, eight
|
||||
/// more characters from it, then the check digit.
|
||||
static DE_ID: LazyLock<Regex> =
|
||||
LazyLock::new(|| re(r"\b[CFGHJKLMNPRTVWXYZ][CFGHJKLMNPRTVWXYZ0-9]{8}\d\b"));
|
||||
|
||||
/// ICAO 9303 check digit: weights 7, 3, 1; letters A=10 … Z=35.
|
||||
pub fn icao_check(chars: &str, check: u32) -> bool {
|
||||
let value = |c: char| c.to_digit(10).unwrap_or_else(|| c as u32 - 'A' as u32 + 10);
|
||||
let sum: u32 = chars
|
||||
.chars()
|
||||
.zip([7, 3, 1].iter().cycle())
|
||||
.map(|(c, w)| value(c) * w)
|
||||
.sum();
|
||||
sum % 10 == check
|
||||
}
|
||||
|
||||
fn de_id_card(text: &str, findings: &mut Findings) {
|
||||
for m in DE_ID.find_iter(text) {
|
||||
let s = m.as_str();
|
||||
if icao_check(&s[..9], num(&s[9..])) {
|
||||
findings.insert(s);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- France ---------------------------------------------------------------
|
||||
|
||||
/// Sex, year, month, department (with Corsica's 2A and 2B), commune, order,
|
||||
/// then the two-digit key, spaces allowed between groups.
|
||||
static FR_NIR: LazyLock<Regex> = LazyLock::new(|| {
|
||||
re(r"\b([1-478]) ?(\d{2}) ?(\d{2}) ?(\d{2}|2[AB]) ?(\d{3}) ?(\d{3}) ?(\d{2})\b")
|
||||
});
|
||||
|
||||
fn fr_nir(text: &str, findings: &mut Findings) {
|
||||
for c in FR_NIR.captures_iter(text) {
|
||||
let month = num(&c[3]);
|
||||
if !(matches!(month, 1..=12 | 20..=42 | 50..=99)) {
|
||||
continue;
|
||||
}
|
||||
let department = match &c[4] {
|
||||
"2A" => "19",
|
||||
"2B" => "18",
|
||||
d => d,
|
||||
};
|
||||
let body = format!(
|
||||
"{}{}{}{}{}{}",
|
||||
&c[1], &c[2], &c[3], department, &c[5], &c[6]
|
||||
);
|
||||
let Ok(value) = body.parse::<u64>() else {
|
||||
continue;
|
||||
};
|
||||
if 97 - value % 97 == u64::from(num(&c[7])) {
|
||||
findings.insert(format!(
|
||||
"{}{}{}{}{}{}{}",
|
||||
&c[1], &c[2], &c[3], &c[4], &c[5], &c[6], &c[7]
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Spain ----------------------------------------------------------------
|
||||
|
||||
static ES_ID: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\b([XYZ]?)[ -]?(\d{7,8})[ -]?([A-Z])\b"));
|
||||
|
||||
const DNI_LETTERS: &[u8] = b"TRWAGMYFPDXBNJZSQVHLCKE";
|
||||
|
||||
fn es_dni_nie(text: &str, findings: &mut Findings) {
|
||||
for c in ES_ID.captures_iter(text) {
|
||||
let prefix = c[1].to_ascii_uppercase();
|
||||
let digits = &c[2];
|
||||
// DNI: eight digits; NIE: X, Y or Z and seven digits
|
||||
let number = match (prefix.as_str(), digits.len()) {
|
||||
("", 8) => digits.to_string(),
|
||||
("X", 7) => format!("0{digits}"),
|
||||
("Y", 7) => format!("1{digits}"),
|
||||
("Z", 7) => format!("2{digits}"),
|
||||
_ => continue,
|
||||
};
|
||||
let letter = c[3].to_ascii_uppercase();
|
||||
if DNI_LETTERS[(num(&number) % 23) as usize] == letter.as_bytes()[0] {
|
||||
findings.insert(format!("{prefix}{digits}{letter}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Italy ----------------------------------------------------------------
|
||||
|
||||
/// Surname and name letters, year, month letter, day, place code, check
|
||||
/// letter; digits may be replaced by letters (omocodia).
|
||||
static IT_CF: LazyLock<Regex> = LazyLock::new(|| {
|
||||
let d = "[0-9LMNPQRSTUV]";
|
||||
re(&format!(
|
||||
r"(?i)\b[A-Z]{{6}}{d}{{2}}[ABCDEHLMPRST]{d}{{2}}[A-Z]{d}{{3}}[A-Z]\b"
|
||||
))
|
||||
});
|
||||
|
||||
/// The Ministry's odd-position values for 0–9 and A–Z.
|
||||
const CF_ODD: [u32; 36] = [
|
||||
1, 0, 5, 7, 9, 13, 15, 17, 19, 21, // 0-9
|
||||
1, 0, 5, 7, 9, 13, 15, 17, 19, 21, 2, 4, 18, 20, 11, 3, 6, 8, 12, 14, 16, 10, 22, 25, 24,
|
||||
23, // A-Z
|
||||
];
|
||||
|
||||
pub fn codice_fiscale_valid(cf: &str) -> bool {
|
||||
let index = |c: u8| {
|
||||
if c.is_ascii_digit() {
|
||||
(c - b'0') as usize
|
||||
} else {
|
||||
(c - b'A') as usize + 10
|
||||
}
|
||||
};
|
||||
let even = |c: u8| {
|
||||
if c.is_ascii_digit() {
|
||||
u32::from(c - b'0')
|
||||
} else {
|
||||
u32::from(c - b'A')
|
||||
}
|
||||
};
|
||||
let bytes = cf.as_bytes();
|
||||
let sum: u32 = bytes[..15]
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, &c)| {
|
||||
if i % 2 == 0 {
|
||||
CF_ODD[index(c)]
|
||||
} else {
|
||||
even(c)
|
||||
}
|
||||
})
|
||||
.sum();
|
||||
u32::from(bytes[15] - b'A') == sum % 26
|
||||
}
|
||||
|
||||
fn it_codice_fiscale(text: &str, findings: &mut Findings) {
|
||||
for m in IT_CF.find_iter(text) {
|
||||
let cf = m.as_str().to_ascii_uppercase();
|
||||
if codice_fiscale_valid(&cf) {
|
||||
findings.insert(cf);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Netherlands ----------------------------------------------------------
|
||||
|
||||
/// Nine digits, sometimes written `1112.22.333`.
|
||||
static NL_BSN: LazyLock<Regex> = LazyLock::new(|| re(r"\b(\d{4})(\.?)(\d{2})(\.?)(\d{3})\b"));
|
||||
|
||||
/// The eleven test: weights 9 down to 2, and −1 for the last digit.
|
||||
pub fn bsn_valid(n: &str) -> bool {
|
||||
let d = digit_values(n);
|
||||
let sum: i64 = d[..8]
|
||||
.iter()
|
||||
.zip((2..=9).rev())
|
||||
.map(|(a, w)| i64::from(a * w))
|
||||
.sum::<i64>()
|
||||
- i64::from(d[8]);
|
||||
sum != 0 && sum % 11 == 0
|
||||
}
|
||||
|
||||
const BSN_WORDS: &[&str] = &[
|
||||
"bsn",
|
||||
"burgerservicenummer",
|
||||
"sofinummer",
|
||||
"sofi-nummer",
|
||||
"citizen service number",
|
||||
];
|
||||
|
||||
fn nl_bsn(text: &str, findings: &mut Findings) {
|
||||
for c in NL_BSN.captures_iter(text) {
|
||||
let whole = c.get(0).unwrap();
|
||||
let n = format!("{}{}{}", &c[1], &c[3], &c[5]);
|
||||
let written = &c[2] == "." && &c[4] == ".";
|
||||
if bsn_valid(&n) && (written || word_near(text, whole.start(), whole.end(), BSN_WORDS)) {
|
||||
findings.insert(n);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Belgium --------------------------------------------------------------
|
||||
|
||||
/// `YY.MM.DD-XXX.CC` or eleven digits.
|
||||
static BE_NN: LazyLock<Regex> =
|
||||
LazyLock::new(|| re(r"\b(\d{2})\.?(\d{2})\.?(\d{2})-?(\d{3})\.?(\d{2})\b"));
|
||||
|
||||
fn be_national_number(text: &str, findings: &mut Findings) {
|
||||
for c in BE_NN.captures_iter(text) {
|
||||
let (month, day) = (num(&c[2]), num(&c[3]));
|
||||
// Month 0 and day 0 mean unknown; bis numbers add 20 or 40 to the month
|
||||
if !(month <= 12 || (20..=32).contains(&month) || (40..=52).contains(&month)) || day > 31 {
|
||||
continue;
|
||||
}
|
||||
let body = format!("{}{}{}{}", &c[1], &c[2], &c[3], &c[4]);
|
||||
let check = u64::from(num(&c[5]));
|
||||
let before_2000 = 97 - body.parse::<u64>().unwrap_or(0) % 97;
|
||||
let since_2000 = 97 - format!("2{body}").parse::<u64>().unwrap_or(0) % 97;
|
||||
if check == before_2000 || check == since_2000 {
|
||||
findings.insert(format!("{body}{}", &c[5]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Poland ---------------------------------------------------------------
|
||||
|
||||
static ELEVEN: LazyLock<Regex> = LazyLock::new(|| re(r"\b\d{11}\b"));
|
||||
|
||||
/// Weights 1, 3, 7, 9 repeating; the birth date encodes the century in the
|
||||
/// month (+80 for the 1800s, +20 for the 2000s, and so on).
|
||||
pub fn pesel_valid(n: &str) -> bool {
|
||||
let d = digit_values(n);
|
||||
let sum: u32 = d[..10]
|
||||
.iter()
|
||||
.zip([1, 3, 7, 9].iter().cycle())
|
||||
.map(|(a, w)| a * w)
|
||||
.sum();
|
||||
let month = d[2] * 10 + d[3];
|
||||
let (century, month) = match month {
|
||||
81..=92 => (1800, month - 80),
|
||||
1..=12 => (1900, month),
|
||||
21..=32 => (2000, month - 20),
|
||||
41..=52 => (2100, month - 40),
|
||||
_ => return false,
|
||||
};
|
||||
let year = century + d[0] * 10 + d[1];
|
||||
(10 - sum % 10) % 10 == d[10] && (1..=super::days_in(year, month)).contains(&(d[4] * 10 + d[5]))
|
||||
}
|
||||
|
||||
const PESEL_WORDS: &[&str] = &["pesel", "numer pesel", "nr pesel"];
|
||||
|
||||
fn pl_pesel(text: &str, findings: &mut Findings) {
|
||||
for m in ELEVEN.find_iter(text) {
|
||||
if pesel_valid(m.as_str()) && word_near(text, m.start(), m.end(), PESEL_WORDS) {
|
||||
findings.insert(m.as_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Sweden ---------------------------------------------------------------
|
||||
|
||||
/// `YYMMDD-NNNN`, `YYYYMMDD-NNNN` (`+` after 100), or the bare digits.
|
||||
static SE_PNR: LazyLock<Regex> =
|
||||
LazyLock::new(|| re(r"\b(?:\d{2})?(\d{2})(\d{2})(\d{2})([-+]?)(\d{4})\b"));
|
||||
|
||||
const SE_WORDS: &[&str] = &[
|
||||
"personnummer",
|
||||
"personnr",
|
||||
"person nr",
|
||||
"samordningsnummer",
|
||||
"pnr",
|
||||
];
|
||||
|
||||
fn se_personnummer(text: &str, findings: &mut Findings) {
|
||||
for c in SE_PNR.captures_iter(text) {
|
||||
let whole = c.get(0).unwrap();
|
||||
let (yy, month, day) = (num(&c[1]), num(&c[2]), num(&c[3]));
|
||||
// Coordination numbers add 60 to the day
|
||||
let day = if day > 60 { day - 60 } else { day };
|
||||
let ten = format!("{}{}{}{}", &c[1], &c[2], &c[3], &c[5]);
|
||||
let written = !c[4].is_empty();
|
||||
if valid_short_date(yy, month, day)
|
||||
&& checks::luhn(&ten)
|
||||
&& (written || word_near(text, whole.start(), whole.end(), SE_WORDS))
|
||||
{
|
||||
findings.insert(ten);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Denmark --------------------------------------------------------------
|
||||
|
||||
static DK_CPR: LazyLock<Regex> = LazyLock::new(|| re(r"\b(\d{2})(\d{2})(\d{2})-?(\d{4})\b"));
|
||||
|
||||
const CPR_WORDS: &[&str] = &["cpr", "cpr-nr", "cpr nr", "cpr-nummer", "personnummer"];
|
||||
|
||||
fn dk_cpr(text: &str, findings: &mut Findings) {
|
||||
for c in DK_CPR.captures_iter(text) {
|
||||
let whole = c.get(0).unwrap();
|
||||
if valid_short_date(num(&c[3]), num(&c[2]), num(&c[1]))
|
||||
&& word_near(text, whole.start(), whole.end(), CPR_WORDS)
|
||||
{
|
||||
findings.insert(whole.as_str().replace('-', ""));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Finland --------------------------------------------------------------
|
||||
|
||||
static FI_HETU: LazyLock<Regex> =
|
||||
LazyLock::new(|| re(r"(?i)\b(\d{2})(\d{2})(\d{2})[-+ABCDEFYXWVU](\d{3})([0-9A-Y])\b"));
|
||||
|
||||
const HETU_CHECK: &[u8] = b"0123456789ABCDEFHJKLMNPRSTUVWXY";
|
||||
|
||||
fn fi_hetu(text: &str, findings: &mut Findings) {
|
||||
for c in FI_HETU.captures_iter(text) {
|
||||
let (day, month, yy) = (num(&c[1]), num(&c[2]), num(&c[3]));
|
||||
let n: u64 = format!("{}{}{}{}", &c[1], &c[2], &c[3], &c[4])
|
||||
.parse()
|
||||
.unwrap_or(0);
|
||||
let check = c[5].to_ascii_uppercase().as_bytes()[0];
|
||||
if valid_short_date(yy, month, day) && HETU_CHECK[(n % 31) as usize] == check {
|
||||
findings.insert(c[0].to_ascii_uppercase());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Ireland --------------------------------------------------------------
|
||||
|
||||
static IE_PPS: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\b(\d{7})([A-W])([ABHW]?)\b"));
|
||||
|
||||
const PPS_CHECK: &[u8] = b"WABCDEFGHIJKLMNOPQRSTUV";
|
||||
|
||||
fn ie_pps(text: &str, findings: &mut Findings) {
|
||||
for c in IE_PPS.captures_iter(text) {
|
||||
let mut sum: u32 = digit_values(&c[1])
|
||||
.iter()
|
||||
.zip((2..=8).rev())
|
||||
.map(|(a, w)| a * w)
|
||||
.sum();
|
||||
// The second letter counts, times 9; W (the old form) counts as 0
|
||||
let second = c[3].to_ascii_uppercase();
|
||||
if let Some(&letter) = second.as_bytes().first()
|
||||
&& letter != b'W'
|
||||
{
|
||||
sum += u32::from(letter - b'A' + 1) * 9;
|
||||
}
|
||||
let check = c[2].to_ascii_uppercase().as_bytes()[0];
|
||||
if PPS_CHECK[(sum % 23) as usize] == check {
|
||||
findings.insert(c[0].to_ascii_uppercase());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Portugal -------------------------------------------------------------
|
||||
|
||||
static NINE: LazyLock<Regex> = LazyLock::new(|| re(r"\b\d{9}\b"));
|
||||
|
||||
/// Mod 11 over weights 9 down to 2; a check of 10 or 11 becomes 0.
|
||||
pub fn nif_valid(n: &str) -> bool {
|
||||
let d = digit_values(n);
|
||||
let sum: u32 = d[..8].iter().zip((2..=9).rev()).map(|(a, w)| a * w).sum();
|
||||
let check = match 11 - sum % 11 {
|
||||
10 | 11 => 0,
|
||||
c => c,
|
||||
};
|
||||
matches!(d[0], 1 | 2 | 3 | 5 | 6 | 8 | 9) && check == d[8]
|
||||
}
|
||||
|
||||
const NIF_WORDS: &[&str] = &[
|
||||
"nif",
|
||||
"contribuinte",
|
||||
"número de identificação fiscal",
|
||||
"numero de contribuinte",
|
||||
];
|
||||
|
||||
fn pt_nif(text: &str, findings: &mut Findings) {
|
||||
for m in NINE.find_iter(text) {
|
||||
if nif_valid(m.as_str()) && word_near(text, m.start(), m.end(), NIF_WORDS) {
|
||||
findings.insert(m.as_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Austria --------------------------------------------------------------
|
||||
|
||||
/// A serial and check digit, then the birth date: `1237 010180`.
|
||||
static AT_SVNR: LazyLock<Regex> = LazyLock::new(|| re(r"\b(\d{3})(\d)( ?)(\d{2})(\d{2})(\d{2})\b"));
|
||||
|
||||
const SVNR_WORDS: &[&str] = &[
|
||||
"sozialversicherungsnummer",
|
||||
"svnr",
|
||||
"sv-nr",
|
||||
"sv-nummer",
|
||||
"versicherungsnummer",
|
||||
];
|
||||
|
||||
fn at_svnr(text: &str, findings: &mut Findings) {
|
||||
for c in AT_SVNR.captures_iter(text) {
|
||||
let whole = c.get(0).unwrap();
|
||||
let n = format!("{}{}{}{}{}", &c[1], &c[2], &c[4], &c[5], &c[6]);
|
||||
let d = digit_values(&n);
|
||||
let sum: u32 = d
|
||||
.iter()
|
||||
.zip([3, 7, 9, 0, 5, 8, 4, 2, 1, 6])
|
||||
.map(|(a, w)| a * w)
|
||||
.sum();
|
||||
let written = &c[3] == " ";
|
||||
if d[0] != 0
|
||||
&& sum % 11 == d[3]
|
||||
&& valid_short_date(num(&c[6]), num(&c[5]), num(&c[4]))
|
||||
&& (written || word_near(text, whole.start(), whole.end(), SVNR_WORDS))
|
||||
&& stands_alone(text, whole.start(), whole.end())
|
||||
{
|
||||
findings.insert(n);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::mailflow::detectors::by_id;
|
||||
|
||||
fn count(id: &str, text: &str) -> usize {
|
||||
by_id(id).unwrap().count(text)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn germany() {
|
||||
assert_eq!(count("de-tax-id", "86 095 742 719"), 1);
|
||||
assert_eq!(count("de-tax-id", "Steuer-ID: 86095742719"), 1);
|
||||
assert_eq!(count("de-tax-id", "Rechnung 86095742719"), 0);
|
||||
assert_eq!(count("de-tax-id", "86 095 742 718"), 0);
|
||||
// ICAO 9303's German specimen card
|
||||
assert_eq!(count("de-id-card", "Ausweis T220001293"), 1);
|
||||
assert_eq!(count("de-id-card", "T220001294"), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn france_spain_italy() {
|
||||
assert_eq!(count("fr-nir", "2 55 08 14 168 025 38"), 1);
|
||||
assert_eq!(count("fr-nir", "255081416802539"), 0);
|
||||
assert_eq!(count("es-dni-nie", "DNI 12345678Z, NIE X-1234567-L"), 2);
|
||||
assert_eq!(count("es-dni-nie", "12345678A"), 0);
|
||||
assert_eq!(count("it-codice-fiscale", "CF: RSSMRA85T10A562S"), 1);
|
||||
assert_eq!(count("it-codice-fiscale", "RSSMRA85T10A562T"), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn benelux() {
|
||||
assert_eq!(count("nl-bsn", "1112.22.333"), 1);
|
||||
assert_eq!(count("nl-bsn", "BSN 111222333"), 1);
|
||||
assert_eq!(count("nl-bsn", "order 111222333"), 0);
|
||||
assert_eq!(count("nl-bsn", "BSN 111222334"), 0);
|
||||
assert_eq!(count("be-national-number", "85.07.30-033.28"), 1);
|
||||
assert_eq!(count("be-national-number", "85073003329"), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nordics() {
|
||||
assert_eq!(count("se-personnummer", "811218-9876"), 1);
|
||||
assert_eq!(count("se-personnummer", "811218-9875"), 0);
|
||||
assert_eq!(count("se-personnummer", "order 8112189876"), 0);
|
||||
assert_eq!(count("se-personnummer", "personnummer 198112189876"), 1);
|
||||
assert_eq!(count("dk-cpr", "CPR-nr: 010170-1234"), 1);
|
||||
assert_eq!(count("dk-cpr", "010170-1234"), 0);
|
||||
assert_eq!(count("dk-cpr", "CPR 320170-1234"), 0);
|
||||
assert_eq!(count("fi-hetu", "131052-308T"), 1);
|
||||
assert_eq!(count("fi-hetu", "131052-308U"), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn poland_ireland_portugal_austria() {
|
||||
assert_eq!(count("pl-pesel", "PESEL 44051401359, pesel 02070803628"), 2);
|
||||
assert_eq!(count("pl-pesel", "PESEL 44051401358"), 0);
|
||||
assert_eq!(count("pl-pesel", "44051401359"), 0);
|
||||
assert_eq!(count("ie-pps", "PPS 1234567T and 1234567FA"), 2);
|
||||
assert_eq!(count("ie-pps", "1234567U"), 0);
|
||||
assert_eq!(count("pt-nif", "NIF 123456789"), 1);
|
||||
assert_eq!(count("pt-nif", "NIF 123456788"), 0);
|
||||
assert_eq!(count("at-svnr", "1237 010180"), 1);
|
||||
assert_eq!(count("at-svnr", "SVNR 1237010180"), 1);
|
||||
assert_eq!(count("at-svnr", "1238 010180"), 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! European identifiers outside the EU (§2.3): Norway's national identity
|
||||
//! number and Switzerland's AHV number.
|
||||
|
||||
use super::{Detector, Findings, Region, Strength, digit_values, valid_short_date};
|
||||
use regex::Regex;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub static DETECTORS: &[Detector] = &[
|
||||
Detector::new(
|
||||
"no-fnr",
|
||||
"Norway: national identity number",
|
||||
Region::Europe,
|
||||
Strength::Checked,
|
||||
no_fnr,
|
||||
),
|
||||
Detector::new(
|
||||
"ch-ahv",
|
||||
"Switzerland: AHV number",
|
||||
Region::Europe,
|
||||
Strength::Checked,
|
||||
ch_ahv,
|
||||
),
|
||||
];
|
||||
|
||||
static ELEVEN: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"\b\d{6} ?\d{5}\b").expect("detector pattern"));
|
||||
|
||||
/// Two mod 11 check digits over a birth date (D-numbers add 40 to the day,
|
||||
/// H-numbers 40 to the month): strong enough to count alone.
|
||||
pub fn fnr_valid(n: &str) -> bool {
|
||||
let d = digit_values(n);
|
||||
if d.len() != 11 {
|
||||
return false;
|
||||
}
|
||||
let check =
|
||||
|weights: &[u32]| match 11 - d.iter().zip(weights).map(|(a, w)| a * w).sum::<u32>() % 11 {
|
||||
11 => Some(0),
|
||||
10 => None,
|
||||
c => Some(c),
|
||||
};
|
||||
let day = d[0] * 10 + d[1];
|
||||
let month = d[2] * 10 + d[3];
|
||||
let day = if day > 40 { day - 40 } else { day };
|
||||
let month = if month > 40 { month - 40 } else { month };
|
||||
valid_short_date(d[4] * 10 + d[5], month, day)
|
||||
&& check(&[3, 7, 6, 1, 8, 9, 4, 5, 2]) == Some(d[9])
|
||||
&& check(&[5, 4, 3, 2, 7, 6, 5, 4, 3, 2]) == Some(d[10])
|
||||
}
|
||||
|
||||
fn no_fnr(text: &str, findings: &mut Findings) {
|
||||
for m in ELEVEN.find_iter(text) {
|
||||
let n = m.as_str().replace(' ', "");
|
||||
if fnr_valid(&n) {
|
||||
findings.insert(n);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `756.1234.5678.97`: the country prefix, then an EAN-13 check digit.
|
||||
static AHV: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r"\b756[. ]?\d{4}[. ]?\d{4}[. ]?\d{2}\b").expect("detector pattern")
|
||||
});
|
||||
|
||||
pub fn ean13_valid(n: &str) -> bool {
|
||||
let d = digit_values(n);
|
||||
if d.len() != 13 {
|
||||
return false;
|
||||
}
|
||||
let sum: u32 = d[..12]
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, x)| if i % 2 == 0 { *x } else { x * 3 })
|
||||
.sum();
|
||||
(10 - sum % 10) % 10 == d[12]
|
||||
}
|
||||
|
||||
fn ch_ahv(text: &str, findings: &mut Findings) {
|
||||
for m in AHV.find_iter(text) {
|
||||
let n: String = m.as_str().chars().filter(char::is_ascii_digit).collect();
|
||||
if ean13_valid(&n) {
|
||||
findings.insert(n);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::mailflow::detectors::by_id;
|
||||
|
||||
fn count(id: &str, text: &str) -> usize {
|
||||
by_id(id).unwrap().count(text)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn norway() {
|
||||
assert_eq!(count("no-fnr", "01019000083"), 1);
|
||||
assert_eq!(count("no-fnr", "010190 00083"), 1);
|
||||
assert_eq!(count("no-fnr", "01019000084"), 0);
|
||||
// Not a date
|
||||
assert_eq!(count("no-fnr", "32019000083"), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn switzerland() {
|
||||
// The federal example
|
||||
assert_eq!(count("ch-ahv", "AHV 756.9217.0769.85"), 1);
|
||||
assert_eq!(count("ch-ahv", "7569217076985"), 1);
|
||||
assert_eq!(count("ch-ahv", "756.9217.0769.86"), 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! Detectors (dlp-and-mail-flow-rules spec, §2.3): each finds one kind of
|
||||
//! identifier in text and reports the distinct ones it found.
|
||||
//!
|
||||
//! A detector is one of two strengths:
|
||||
//!
|
||||
//! - **Checked**: the identifier carries a published check digit or
|
||||
//! checksum, so a random number rarely passes; found on its own.
|
||||
//! - **Needs a word**: the format alone is too common, so a candidate counts
|
||||
//! only with a corroborating word within [`WINDOW`] characters either
|
||||
//! side.
|
||||
//!
|
||||
//! Findings are distinct normalized values (digits only, upper case), so the
|
||||
//! same card number pasted twice counts once. They stay in memory: callers
|
||||
//! read only [`Findings::len`].
|
||||
|
||||
pub mod africa;
|
||||
pub mod americas;
|
||||
pub mod any;
|
||||
pub mod asia;
|
||||
pub mod australia;
|
||||
pub mod canada;
|
||||
pub mod checks;
|
||||
pub mod eu;
|
||||
pub mod europe;
|
||||
pub mod templates;
|
||||
pub mod uk;
|
||||
pub mod us;
|
||||
|
||||
use ahash::AHashSet;
|
||||
|
||||
/// How far, in characters, a corroborating word may be from a candidate.
|
||||
pub const WINDOW: usize = 50;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Strength {
|
||||
Checked,
|
||||
NeedsWord,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Region {
|
||||
Any,
|
||||
Us,
|
||||
Uk,
|
||||
Canada,
|
||||
Australia,
|
||||
Eu,
|
||||
Europe,
|
||||
Asia,
|
||||
Americas,
|
||||
Africa,
|
||||
}
|
||||
|
||||
/// The distinct values one detector found.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct Findings(AHashSet<String>);
|
||||
|
||||
impl Findings {
|
||||
pub fn insert(&mut self, value: impl Into<String>) {
|
||||
self.0.insert(value.into());
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
self.0.len()
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.0.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Detector {
|
||||
/// Stable id, stored in rules: `payment-card`, `iban`, `us-ssn`.
|
||||
pub id: &'static str,
|
||||
pub name: &'static str,
|
||||
pub region: Region,
|
||||
pub strength: Strength,
|
||||
find: fn(&str, &mut Findings),
|
||||
}
|
||||
|
||||
impl Detector {
|
||||
pub const fn new(
|
||||
id: &'static str,
|
||||
name: &'static str,
|
||||
region: Region,
|
||||
strength: Strength,
|
||||
find: fn(&str, &mut Findings),
|
||||
) -> Self {
|
||||
Self {
|
||||
id,
|
||||
name,
|
||||
region,
|
||||
strength,
|
||||
find,
|
||||
}
|
||||
}
|
||||
|
||||
/// Adds what this detector finds in `text` to `findings`. Call once per
|
||||
/// piece of text (subject, each part, each attachment) with the same
|
||||
/// `findings`, then read its length.
|
||||
pub fn find(&self, text: &str, findings: &mut Findings) {
|
||||
(self.find)(text, findings)
|
||||
}
|
||||
|
||||
/// The distinct values found in one text.
|
||||
pub fn count(&self, text: &str) -> usize {
|
||||
let mut findings = Findings::default();
|
||||
self.find(text, &mut findings);
|
||||
findings.len()
|
||||
}
|
||||
}
|
||||
|
||||
/// Every detector, in the order the console lists them.
|
||||
pub fn all() -> impl Iterator<Item = &'static Detector> {
|
||||
[
|
||||
any::DETECTORS,
|
||||
us::DETECTORS,
|
||||
uk::DETECTORS,
|
||||
canada::DETECTORS,
|
||||
australia::DETECTORS,
|
||||
eu::DETECTORS,
|
||||
europe::DETECTORS,
|
||||
asia::DETECTORS,
|
||||
americas::DETECTORS,
|
||||
africa::DETECTORS,
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
}
|
||||
|
||||
pub fn by_id(id: &str) -> Option<&'static Detector> {
|
||||
all().find(|detector| detector.id == id)
|
||||
}
|
||||
|
||||
/// Whether one of `words` appears, as a whole word and ignoring case, within
|
||||
/// [`WINDOW`] characters before `start` or after `end` (byte offsets of the
|
||||
/// candidate in `text`). The window is widened by the longest word, so a
|
||||
/// word that reaches into it still counts whole.
|
||||
pub fn word_near(text: &str, start: usize, end: usize, words: &[&str]) -> bool {
|
||||
let reach = WINDOW + words.iter().map(|w| w.chars().count()).max().unwrap_or(0);
|
||||
let before = text[..start]
|
||||
.char_indices()
|
||||
.rev()
|
||||
.nth(reach - 1)
|
||||
.map_or(0, |(i, _)| i);
|
||||
let after = text[end..]
|
||||
.char_indices()
|
||||
.nth(reach)
|
||||
.map_or(text.len(), |(i, _)| end + i);
|
||||
let window = text[before..after].to_lowercase();
|
||||
words.iter().any(|word| contains_word(&window, word))
|
||||
}
|
||||
|
||||
/// Whether `word` (lower case) appears in `haystack` (lower case) with no
|
||||
/// letter or digit on either side.
|
||||
pub fn contains_word(haystack: &str, word: &str) -> bool {
|
||||
haystack.match_indices(word).any(|(i, _)| {
|
||||
let before_ok = haystack[..i]
|
||||
.chars()
|
||||
.next_back()
|
||||
.is_none_or(|c| !c.is_alphanumeric());
|
||||
let after_ok = haystack[i + word.len()..]
|
||||
.chars()
|
||||
.next()
|
||||
.is_none_or(|c| !c.is_alphanumeric());
|
||||
before_ok && after_ok
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether the match at `start..end` stands alone: no digit or letter
|
||||
/// directly before or after it, so `123-45-6789` isn't found inside a
|
||||
/// longer run of digits.
|
||||
pub fn stands_alone(text: &str, start: usize, end: usize) -> bool {
|
||||
let before = text[..start].chars().next_back();
|
||||
let after = text[end..].chars().next();
|
||||
before.is_none_or(|c| !c.is_alphanumeric()) && after.is_none_or(|c| !c.is_alphanumeric())
|
||||
}
|
||||
|
||||
/// Days in `month` of `year` (0 for a month that doesn't exist).
|
||||
pub fn days_in(year: u32, month: u32) -> u32 {
|
||||
match month {
|
||||
1 | 3 | 5 | 7 | 8 | 10 | 12 => 31,
|
||||
4 | 6 | 9 | 11 => 30,
|
||||
2 if year.is_multiple_of(4) && (!year.is_multiple_of(100) || year.is_multiple_of(400)) => {
|
||||
29
|
||||
}
|
||||
2 => 28,
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `year`-`month`-`day` is a real date between 1900 and 2100.
|
||||
pub fn valid_date(year: u32, month: u32, day: u32) -> bool {
|
||||
(1900..=2100).contains(&year) && (1..=days_in(year, month)).contains(&day)
|
||||
}
|
||||
|
||||
/// Whether a two-digit year, month and day make a real date in either the
|
||||
/// 1900s or the 2000s.
|
||||
pub fn valid_short_date(yy: u32, month: u32, day: u32) -> bool {
|
||||
valid_date(1900 + yy, month, day) || valid_date(2000 + yy, month, day)
|
||||
}
|
||||
|
||||
/// The value of each digit in `s`.
|
||||
pub fn digit_values(s: &str) -> Vec<u32> {
|
||||
s.bytes()
|
||||
.filter(u8::is_ascii_digit)
|
||||
.map(|b| u32::from(b - b'0'))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The ASCII digits of `s`.
|
||||
pub fn digits(s: &str) -> String {
|
||||
s.chars().filter(char::is_ascii_digit).collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn words_are_whole_and_near() {
|
||||
let text = "Your passport number is X1234567, thanks";
|
||||
let start = text.find("X123").unwrap();
|
||||
assert!(word_near(text, start, start + 8, &["passport"]));
|
||||
assert!(!word_near(text, start, start + 8, &["pass"]));
|
||||
let far = format!("passport{}X1234567", " ".repeat(60));
|
||||
let start = far.find("X123").unwrap();
|
||||
assert!(!word_near(&far, start, start + 8, &["passport"]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn near_counts_characters_not_bytes() {
|
||||
// 45 two-byte characters between the word and the candidate: within
|
||||
// 50 characters, though over 50 bytes
|
||||
let text = format!("passport {} X1234567", "é".repeat(45));
|
||||
let start = text.find("X123").unwrap();
|
||||
assert!(word_near(&text, start, start + 8, &["passport"]));
|
||||
}
|
||||
|
||||
/// An ordinary business email: order, invoice and tracking numbers,
|
||||
/// dates, amounts, a street address. Nothing here is an identifier, so
|
||||
/// no detector may fire, except the contact ones on the signature.
|
||||
#[test]
|
||||
fn ordinary_mail_finds_nothing() {
|
||||
let text = "Hi Dana,\n\nThanks for order 4471-2290 placed 2026-09-14. Invoice INV-2026-00917 \
|
||||
for $12,480.00 is due 10/31/2026; PO 7731902 covers lines 1-14. Tracking \
|
||||
1Z999AA10123456784, parcel 3 of 5, 12.5 kg, box 40x30x20 cm. Meeting moved to \
|
||||
Tuesday 9:30-10:15 in room 2B, building 1177. Ticket #5520318, case 20260914-0042. \
|
||||
Version 2026.9.28.4, build 118822, commit 5a73a118. Serial SN-88213-X. \
|
||||
Ship to 1600 Amphitheatre Pkwy, Mountain View, CA 94043. Revenue grew 18% to \
|
||||
1,204,332 units; see figures 3.1-3.4 and table 12.\n\nBest,\nSam\n\
|
||||
Sam Rivera | +1 (415) 555-2671 | [email protected]";
|
||||
let quiet = ["email-addresses", "phone-numbers"];
|
||||
for detector in all().filter(|d| !quiet.contains(&d.id)) {
|
||||
assert_eq!(
|
||||
detector.count(text),
|
||||
0,
|
||||
"{} fired on ordinary mail",
|
||||
detector.id
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ids_are_unique() {
|
||||
let mut seen = AHashSet::new();
|
||||
for detector in all() {
|
||||
assert!(seen.insert(detector.id), "duplicate id {}", detector.id);
|
||||
assert!(by_id(detector.id).is_some());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! Templates (§2.3): named sets of detectors, so a policy doesn't pick forty
|
||||
//! one at a time. Each is named for what it finds, never for a law, and is a
|
||||
//! starting point: once added to a rule, its detectors can be changed.
|
||||
|
||||
pub struct Template {
|
||||
pub id: &'static str,
|
||||
pub name: &'static str,
|
||||
pub detectors: &'static [&'static str],
|
||||
}
|
||||
|
||||
pub static TEMPLATES: &[Template] = &[
|
||||
Template {
|
||||
id: "payment-and-bank",
|
||||
name: "Payment cards and bank accounts",
|
||||
detectors: &["payment-card", "iban", "swift-bic", "us-aba-routing"],
|
||||
},
|
||||
Template {
|
||||
id: "us-personal",
|
||||
name: "US personal identifiers",
|
||||
detectors: &[
|
||||
"us-ssn",
|
||||
"us-itin",
|
||||
"us-ein",
|
||||
"us-drivers-license",
|
||||
"passport",
|
||||
"date-of-birth",
|
||||
],
|
||||
},
|
||||
Template {
|
||||
id: "uk-personal",
|
||||
name: "UK personal identifiers",
|
||||
detectors: &["uk-nino", "uk-utr", "uk-nhs", "passport", "date-of-birth"],
|
||||
},
|
||||
Template {
|
||||
id: "eu-national",
|
||||
name: "EU national identifiers",
|
||||
detectors: &[
|
||||
"de-tax-id",
|
||||
"de-id-card",
|
||||
"fr-nir",
|
||||
"es-dni-nie",
|
||||
"it-codice-fiscale",
|
||||
"nl-bsn",
|
||||
"be-national-number",
|
||||
"pl-pesel",
|
||||
"se-personnummer",
|
||||
"dk-cpr",
|
||||
"fi-hetu",
|
||||
"ie-pps",
|
||||
"pt-nif",
|
||||
"at-svnr",
|
||||
],
|
||||
},
|
||||
Template {
|
||||
id: "health",
|
||||
name: "Health identifiers",
|
||||
detectors: &["uk-nhs", "us-mbi", "us-npi", "us-dea", "au-medicare"],
|
||||
},
|
||||
Template {
|
||||
id: "credentials",
|
||||
name: "Credentials and keys",
|
||||
detectors: &["private-key", "credentials"],
|
||||
},
|
||||
Template {
|
||||
id: "contact-lists",
|
||||
name: "Contact lists",
|
||||
detectors: &["email-addresses", "phone-numbers"],
|
||||
},
|
||||
];
|
||||
|
||||
pub fn by_id(id: &str) -> Option<&'static Template> {
|
||||
TEMPLATES.iter().find(|template| template.id == id)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn every_template_names_real_detectors() {
|
||||
for template in TEMPLATES {
|
||||
for id in template.detectors {
|
||||
assert!(
|
||||
super::super::by_id(id).is_some(),
|
||||
"{}: no detector {id}",
|
||||
template.id
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! United Kingdom identifiers (§2.3): HMRC's National Insurance number and
|
||||
//! Unique Taxpayer Reference, and the NHS number.
|
||||
|
||||
use super::{Detector, Findings, Region, Strength, word_near};
|
||||
use regex::Regex;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub static DETECTORS: &[Detector] = &[
|
||||
Detector::new(
|
||||
"uk-nino",
|
||||
"UK National Insurance number",
|
||||
Region::Uk,
|
||||
Strength::Checked,
|
||||
nino,
|
||||
),
|
||||
Detector::new(
|
||||
"uk-nhs",
|
||||
"UK NHS number",
|
||||
Region::Uk,
|
||||
Strength::Checked,
|
||||
nhs,
|
||||
),
|
||||
Detector::new(
|
||||
"uk-utr",
|
||||
"UK Unique Taxpayer Reference",
|
||||
Region::Uk,
|
||||
Strength::NeedsWord,
|
||||
utr,
|
||||
),
|
||||
];
|
||||
|
||||
fn re(pattern: &str) -> Regex {
|
||||
Regex::new(pattern).expect("detector pattern")
|
||||
}
|
||||
|
||||
/// Two letters, six digits (often in pairs), a suffix A–D.
|
||||
static NINO: LazyLock<Regex> =
|
||||
LazyLock::new(|| re(r"(?i)\b([A-Z])([A-Z]) ?(\d{2}) ?(\d{2}) ?(\d{2}) ?([A-D])\b"));
|
||||
|
||||
/// HMRC's rules: D, F, I, Q, U and V are never used; O never second; and
|
||||
/// BG, GB, KN, NK, NT, TN and ZZ are never allocated.
|
||||
fn nino_prefix(first: char, second: char) -> bool {
|
||||
const NEVER: &str = "DFIQUV";
|
||||
let pair: String = [first, second].iter().collect();
|
||||
!NEVER.contains(first)
|
||||
&& !NEVER.contains(second)
|
||||
&& second != 'O'
|
||||
&& !["BG", "GB", "KN", "NK", "NT", "TN", "ZZ"].contains(&pair.as_str())
|
||||
}
|
||||
|
||||
fn nino(text: &str, findings: &mut Findings) {
|
||||
for c in NINO.captures_iter(text) {
|
||||
let first = c[1].to_ascii_uppercase().chars().next().unwrap();
|
||||
let second = c[2].to_ascii_uppercase().chars().next().unwrap();
|
||||
if nino_prefix(first, second) {
|
||||
findings.insert(format!(
|
||||
"{first}{second}{}{}{}{}",
|
||||
&c[3],
|
||||
&c[4],
|
||||
&c[5],
|
||||
c[6].to_ascii_uppercase()
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `NNN NNN NNNN` stands alone; ten bare digits need a word.
|
||||
static NHS: LazyLock<Regex> = LazyLock::new(|| re(r"\b(\d{3})([ -]?)(\d{3})([ -]?)(\d{4})\b"));
|
||||
|
||||
/// Mod 11: weights 10 down to 2 over the first nine digits; the check digit
|
||||
/// is 11 minus the remainder (11 becomes 0; 10 is never issued).
|
||||
pub fn nhs_valid(n: &str) -> bool {
|
||||
let d: Vec<u32> = n.bytes().map(|b| u32::from(b - b'0')).collect();
|
||||
let sum: u32 = d[..9].iter().zip((2..=10).rev()).map(|(a, w)| a * w).sum();
|
||||
match 11 - sum % 11 {
|
||||
11 => d[9] == 0,
|
||||
10 => false,
|
||||
check => d[9] == check,
|
||||
}
|
||||
}
|
||||
|
||||
const NHS_WORDS: &[&str] = &["nhs", "nhs number", "nhs no"];
|
||||
|
||||
fn nhs(text: &str, findings: &mut Findings) {
|
||||
for c in NHS.captures_iter(text) {
|
||||
let whole = c.get(0).unwrap();
|
||||
let n = format!("{}{}{}", &c[1], &c[3], &c[5]);
|
||||
let written = !c[2].is_empty() && c[2] == c[4];
|
||||
if nhs_valid(&n) && (written || word_near(text, whole.start(), whole.end(), NHS_WORDS)) {
|
||||
findings.insert(n);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static UTR: LazyLock<Regex> = LazyLock::new(|| re(r"\b\d{5} ?\d{5}\b"));
|
||||
|
||||
const UTR_WORDS: &[&str] = &[
|
||||
"utr",
|
||||
"unique taxpayer reference",
|
||||
"tax reference",
|
||||
"self assessment",
|
||||
];
|
||||
|
||||
fn utr(text: &str, findings: &mut Findings) {
|
||||
for m in UTR.find_iter(text) {
|
||||
if word_near(text, m.start(), m.end(), UTR_WORDS) {
|
||||
findings.insert(m.as_str().replace(' ', ""));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::mailflow::detectors::by_id;
|
||||
|
||||
fn count(id: &str, text: &str) -> usize {
|
||||
by_id(id).unwrap().count(text)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn national_insurance() {
|
||||
assert_eq!(count("uk-nino", "NI: AB 12 34 56 C, ce123456d"), 2);
|
||||
// Letters never used, pairs never allocated, a suffix past D
|
||||
for bad in [
|
||||
"QQ123456C",
|
||||
"AO123456C",
|
||||
"GB123456A",
|
||||
"AB123456E",
|
||||
"DA123456A",
|
||||
] {
|
||||
assert_eq!(count("uk-nino", bad), 0, "{bad}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nhs_numbers() {
|
||||
// The NHS's own example
|
||||
assert_eq!(count("uk-nhs", "943 476 5919"), 1);
|
||||
assert_eq!(count("uk-nhs", "943 476 5918"), 0);
|
||||
assert_eq!(count("uk-nhs", "order 9434765919"), 0);
|
||||
assert_eq!(count("uk-nhs", "NHS number 9434765919"), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn utr() {
|
||||
assert_eq!(count("uk-utr", "UTR 12345 67890"), 1);
|
||||
assert_eq!(count("uk-utr", "order 1234567890"), 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,304 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! United States identifiers (§2.3), each from its issuer's published rules:
|
||||
//! the SSA (SSN), the IRS (ITIN, EIN), the ABA (routing numbers), CMS (MBI,
|
||||
//! NPI) and the DEA.
|
||||
|
||||
use super::{Detector, Findings, Region, Strength, checks, digits, stands_alone, word_near};
|
||||
use regex::Regex;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub static DETECTORS: &[Detector] = &[
|
||||
Detector::new(
|
||||
"us-ssn",
|
||||
"US Social Security number",
|
||||
Region::Us,
|
||||
Strength::Checked,
|
||||
ssn,
|
||||
),
|
||||
Detector::new("us-itin", "US ITIN", Region::Us, Strength::Checked, itin),
|
||||
Detector::new("us-ein", "US EIN", Region::Us, Strength::NeedsWord, ein),
|
||||
Detector::new(
|
||||
"us-aba-routing",
|
||||
"US bank routing number",
|
||||
Region::Us,
|
||||
Strength::NeedsWord,
|
||||
aba_routing,
|
||||
),
|
||||
Detector::new(
|
||||
"us-drivers-license",
|
||||
"US driver's license",
|
||||
Region::Us,
|
||||
Strength::NeedsWord,
|
||||
drivers_license,
|
||||
),
|
||||
Detector::new(
|
||||
"us-mbi",
|
||||
"US Medicare Beneficiary Identifier",
|
||||
Region::Us,
|
||||
Strength::Checked,
|
||||
mbi,
|
||||
),
|
||||
Detector::new(
|
||||
"us-npi",
|
||||
"US National Provider Identifier",
|
||||
Region::Us,
|
||||
Strength::NeedsWord,
|
||||
npi,
|
||||
),
|
||||
Detector::new(
|
||||
"us-dea",
|
||||
"US DEA registration number",
|
||||
Region::Us,
|
||||
Strength::Checked,
|
||||
dea,
|
||||
),
|
||||
];
|
||||
|
||||
fn re(pattern: &str) -> Regex {
|
||||
Regex::new(pattern).expect("detector pattern")
|
||||
}
|
||||
|
||||
/// `AAA-GG-SSSS` (dashes or spaces), or nine bare digits.
|
||||
static NINE: LazyLock<Regex> = LazyLock::new(|| re(r"\b(\d{3})([ -]?)(\d{2})([ -]?)(\d{4})\b"));
|
||||
|
||||
/// Numbers the SSA has published as never valid: widely printed examples.
|
||||
const SSN_EXAMPLES: &[&str] = &["078051120", "219099999"];
|
||||
|
||||
fn ssn_rules(area: u32, group: u32, serial: u32) -> bool {
|
||||
area != 0 && area != 666 && area < 900 && group != 0 && serial != 0
|
||||
}
|
||||
|
||||
const SSN_WORDS: &[&str] = &["ssn", "social security", "soc sec", "ss#", "ss no"];
|
||||
|
||||
fn ssn(text: &str, findings: &mut Findings) {
|
||||
for c in NINE.captures_iter(text) {
|
||||
let whole = c.get(0).unwrap();
|
||||
let (area, group, serial) = (num(&c[1]), num(&c[3]), num(&c[5]));
|
||||
let number = format!("{}{}{}", &c[1], &c[3], &c[5]);
|
||||
// Written form (with both separators, the same one) stands alone;
|
||||
// nine bare digits need a word
|
||||
let written = !c[2].is_empty() && c[2] == c[4];
|
||||
if ssn_rules(area, group, serial)
|
||||
&& !SSN_EXAMPLES.contains(&number.as_str())
|
||||
&& (written || word_near(text, whole.start(), whole.end(), SSN_WORDS))
|
||||
{
|
||||
findings.insert(number);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// ITINs: 9XX, then a group in the IRS's ranges.
|
||||
fn itin_group(group: u32) -> bool {
|
||||
matches!(group, 50..=65 | 70..=88 | 90..=92 | 94..=99)
|
||||
}
|
||||
|
||||
const ITIN_WORDS: &[&str] = &["itin", "taxpayer identification", "tax id"];
|
||||
|
||||
fn itin(text: &str, findings: &mut Findings) {
|
||||
for c in NINE.captures_iter(text) {
|
||||
let whole = c.get(0).unwrap();
|
||||
let written = !c[2].is_empty() && c[2] == c[4];
|
||||
if c[1].starts_with('9')
|
||||
&& itin_group(num(&c[3]))
|
||||
&& (written || word_near(text, whole.start(), whole.end(), ITIN_WORDS))
|
||||
{
|
||||
findings.insert(format!("{}{}{}", &c[1], &c[3], &c[5]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static EIN: LazyLock<Regex> = LazyLock::new(|| re(r"\b(\d{2})-?(\d{7})\b"));
|
||||
|
||||
/// The prefixes the IRS assigns to its campuses and internet EINs.
|
||||
fn ein_prefix(prefix: u32) -> bool {
|
||||
matches!(prefix, 1..=6 | 10..=16 | 20..=27 | 30..=48 | 50..=68 | 71..=77 | 80..=88 | 90..=95 | 98 | 99)
|
||||
}
|
||||
|
||||
const EIN_WORDS: &[&str] = &[
|
||||
"ein",
|
||||
"fein",
|
||||
"employer identification",
|
||||
"tax id",
|
||||
"tin",
|
||||
"federal tax",
|
||||
];
|
||||
|
||||
fn ein(text: &str, findings: &mut Findings) {
|
||||
for c in EIN.captures_iter(text) {
|
||||
let whole = c.get(0).unwrap();
|
||||
if ein_prefix(num(&c[1])) && word_near(text, whole.start(), whole.end(), EIN_WORDS) {
|
||||
findings.insert(format!("{}{}", &c[1], &c[2]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static ROUTING: LazyLock<Regex> = LazyLock::new(|| re(r"\b\d{9}\b"));
|
||||
|
||||
/// The ABA check: 3, 7 and 1 weights, mod 10; and a Federal Reserve prefix.
|
||||
pub fn aba_valid(n: &str) -> bool {
|
||||
let d: Vec<u32> = n.bytes().map(|b| u32::from(b - b'0')).collect();
|
||||
let prefix = d[0] * 10 + d[1];
|
||||
matches!(prefix, 0..=12 | 21..=32 | 61..=72 | 80)
|
||||
&& (3 * (d[0] + d[3] + d[6]) + 7 * (d[1] + d[4] + d[7]) + (d[2] + d[5] + d[8]))
|
||||
.is_multiple_of(10)
|
||||
}
|
||||
|
||||
const ROUTING_WORDS: &[&str] = &["routing", "aba", "rtn", "routing number", "transit"];
|
||||
|
||||
fn aba_routing(text: &str, findings: &mut Findings) {
|
||||
for m in ROUTING.find_iter(text) {
|
||||
// One random number in ten passes the check: always needs a word
|
||||
if aba_valid(m.as_str()) && word_near(text, m.start(), m.end(), ROUTING_WORDS) {
|
||||
findings.insert(m.as_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The shapes states issue: up to two letters, then 5–14 digits, dashes
|
||||
/// allowed (Florida and Illinois print them).
|
||||
static LICENSE: LazyLock<Regex> = LazyLock::new(|| re(r"\b[A-Z]{0,2}\d[\d-]{3,16}\d\b"));
|
||||
|
||||
const LICENSE_WORDS: &[&str] = &[
|
||||
"driver's license",
|
||||
"drivers license",
|
||||
"driver license",
|
||||
"driver's licence",
|
||||
"dl",
|
||||
"dl#",
|
||||
"license number",
|
||||
"lic no",
|
||||
"dmv",
|
||||
];
|
||||
|
||||
fn drivers_license(text: &str, findings: &mut Findings) {
|
||||
for m in LICENSE.find_iter(text) {
|
||||
let n = digits(m.as_str());
|
||||
if (5..=14).contains(&n.len()) && word_near(text, m.start(), m.end(), LICENSE_WORDS) {
|
||||
findings.insert(m.as_str().replace('-', ""));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// CMS's MBI: 11 characters in a fixed pattern of digits, letters and
|
||||
/// either, the letters S, L, O, I, B and Z never used; dashes may follow the
|
||||
/// 4th and 7th.
|
||||
static MBI: LazyLock<Regex> = LazyLock::new(|| {
|
||||
let c = "[AC-HJKMNP-RT-Y]";
|
||||
let an = "[AC-HJKMNP-RT-Y0-9]";
|
||||
re(&format!(
|
||||
r"\b[1-9]{c}{an}[0-9]-?{c}{an}[0-9]-?{c}{c}[0-9][0-9]\b"
|
||||
))
|
||||
});
|
||||
|
||||
fn mbi(text: &str, findings: &mut Findings) {
|
||||
for m in MBI.find_iter(text) {
|
||||
findings.insert(m.as_str().replace('-', ""));
|
||||
}
|
||||
}
|
||||
|
||||
static TEN: LazyLock<Regex> = LazyLock::new(|| re(r"\b[12]\d{9}\b"));
|
||||
|
||||
const NPI_WORDS: &[&str] = &["npi", "national provider", "provider id", "provider number"];
|
||||
|
||||
/// NPI: Luhn over the ISO card-issuer prefix 80840 and the number.
|
||||
fn npi(text: &str, findings: &mut Findings) {
|
||||
for m in TEN.find_iter(text) {
|
||||
if checks::luhn(&format!("80840{}", m.as_str()))
|
||||
&& word_near(text, m.start(), m.end(), NPI_WORDS)
|
||||
{
|
||||
findings.insert(m.as_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static DEA: LazyLock<Regex> = LazyLock::new(|| re(r"\b([ABCDEFGHJKLMPRSTUX][A-Z9])(\d{7})\b"));
|
||||
|
||||
/// DEA: (1st + 3rd + 5th) + 2 × (2nd + 4th + 6th) ends in the 7th digit.
|
||||
fn dea(text: &str, findings: &mut Findings) {
|
||||
for c in DEA.captures_iter(text) {
|
||||
let d: Vec<u32> = c[2].bytes().map(|b| u32::from(b - b'0')).collect();
|
||||
if ((d[0] + d[2] + d[4]) + 2 * (d[1] + d[3] + d[5])) % 10 == d[6] {
|
||||
let whole = c.get(0).unwrap();
|
||||
if stands_alone(text, whole.start(), whole.end()) {
|
||||
findings.insert(whole.as_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn num(s: &str) -> u32 {
|
||||
s.parse().unwrap_or(0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::mailflow::detectors::by_id;
|
||||
|
||||
fn count(id: &str, text: &str) -> usize {
|
||||
by_id(id).unwrap().count(text)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssn() {
|
||||
assert_eq!(count("us-ssn", "SSN 536-22-1234, also 536 22 1235"), 2);
|
||||
// Bare digits: only with a word
|
||||
assert_eq!(count("us-ssn", "ref 536221234"), 0);
|
||||
assert_eq!(count("us-ssn", "social security: 536221234"), 1);
|
||||
// Never issued, the SSA's printed examples, mixed separators
|
||||
for bad in [
|
||||
"000-12-3456",
|
||||
"666-12-3456",
|
||||
"912-12-3456",
|
||||
"123-00-4567",
|
||||
"123-45-0000",
|
||||
"078-05-1120",
|
||||
"536-22 1234",
|
||||
] {
|
||||
assert_eq!(count("us-ssn", bad), 0, "{bad}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn itin_and_ein() {
|
||||
assert_eq!(count("us-itin", "912-70-1234"), 1);
|
||||
assert_eq!(count("us-itin", "912-69-1234"), 0);
|
||||
assert_eq!(count("us-ssn", "912-70-1234"), 0);
|
||||
assert_eq!(count("us-ein", "EIN: 12-3456789"), 1);
|
||||
assert_eq!(count("us-ein", "part 12-3456789"), 0);
|
||||
assert_eq!(count("us-ein", "EIN 07-3456789"), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn routing_needs_a_word() {
|
||||
assert_eq!(count("us-aba-routing", "Routing number 011000015"), 1);
|
||||
assert_eq!(count("us-aba-routing", "ABA 021000021"), 1);
|
||||
assert_eq!(count("us-aba-routing", "invoice 011000015"), 0);
|
||||
assert_eq!(count("us-aba-routing", "routing 011000016"), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn licenses() {
|
||||
assert_eq!(count("us-drivers-license", "Driver's license: D1234567"), 1);
|
||||
assert_eq!(count("us-drivers-license", "DL# S123-456-78-901-0"), 1);
|
||||
assert_eq!(count("us-drivers-license", "Order D1234567"), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn health_identifiers() {
|
||||
// CMS's own MBI example
|
||||
assert_eq!(count("us-mbi", "Medicare 1EG4-TE5-MK73"), 1);
|
||||
assert_eq!(count("us-mbi", "1EG4TE5MK73"), 1);
|
||||
assert_eq!(count("us-mbi", "1EG4-TE5-MK7S"), 0);
|
||||
// CMS's NPI example
|
||||
assert_eq!(count("us-npi", "NPI 1234567893"), 1);
|
||||
assert_eq!(count("us-npi", "NPI 1234567894"), 0);
|
||||
assert_eq!(count("us-npi", "call 1234567893"), 0);
|
||||
assert_eq!(count("us-dea", "DEA AB1234563"), 1);
|
||||
assert_eq!(count("us-dea", "AB1234564"), 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,697 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! Evaluating rules against a message (§2.1–§2.4). Rules are compiled once,
|
||||
//! when they change: word lists become automata, patterns regexes. A message
|
||||
//! is then checked against every enabled rule in order; each detector runs
|
||||
//! at most once per message, and only when some rule asks for it.
|
||||
//!
|
||||
//! Pure: the caller parses the message, extracts attachment text
|
||||
//! ([`super::extract`]) and knows the sender's groups and tenant. What comes
|
||||
//! back is which rules matched, with each detector's count, and what DLP
|
||||
//! decided; the matched text itself never leaves here (§2.7).
|
||||
|
||||
use super::{
|
||||
detectors::{self, Findings},
|
||||
extract::Extracted,
|
||||
rules::{Action, Condition, Direction, Kind, Rule},
|
||||
words::{Pattern, WordList},
|
||||
};
|
||||
use ahash::AHashMap;
|
||||
use std::borrow::Cow;
|
||||
|
||||
/// Who sent a message, and to whom.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct Envelope<'a> {
|
||||
/// Outgoing (an authenticated sender) or incoming.
|
||||
pub outgoing: bool,
|
||||
pub sender: &'a str,
|
||||
pub sender_groups: &'a [u32],
|
||||
pub sender_tenant: Option<u32>,
|
||||
pub recipients: Vec<Recipient<'a>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct Recipient<'a> {
|
||||
pub address: &'a str,
|
||||
/// At a domain this server hosts.
|
||||
pub local: bool,
|
||||
pub groups: &'a [u32],
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Attachment<'a> {
|
||||
pub name: Option<&'a str>,
|
||||
/// Declared type, or detected where the caller knows better.
|
||||
pub content_type: Cow<'a, str>,
|
||||
pub size: u64,
|
||||
pub extracted: Extracted,
|
||||
}
|
||||
|
||||
/// What rules look at.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct Content<'a> {
|
||||
pub subject: &'a str,
|
||||
/// Each text and HTML part, as text.
|
||||
pub bodies: Vec<Cow<'a, str>>,
|
||||
pub headers: Vec<(&'a str, &'a str)>,
|
||||
pub attachments: Vec<Attachment<'a>>,
|
||||
pub size: u64,
|
||||
/// Text past the inspection limit wasn't read.
|
||||
pub truncated: bool,
|
||||
}
|
||||
|
||||
impl Content<'_> {
|
||||
fn texts(&self) -> impl Iterator<Item = &str> {
|
||||
std::iter::once(self.subject)
|
||||
.chain(self.bodies.iter().map(|b| b.as_ref()))
|
||||
.chain(self.attachments.iter().filter_map(|a| match &a.extracted {
|
||||
Extracted::Text(text) => Some(text.as_str()),
|
||||
_ => None,
|
||||
}))
|
||||
}
|
||||
|
||||
fn cant_be_inspected(&self) -> bool {
|
||||
self.truncated
|
||||
|| self
|
||||
.attachments
|
||||
.iter()
|
||||
.any(|a| matches!(a.extracted, Extracted::NotInspectable(_)))
|
||||
}
|
||||
}
|
||||
|
||||
enum Check {
|
||||
Plain(Condition),
|
||||
Words(WordList, u32),
|
||||
Pattern(Pattern, u32),
|
||||
Header {
|
||||
name: String,
|
||||
contains: Option<String>,
|
||||
matches: Option<Pattern>,
|
||||
},
|
||||
AttachmentName(Pattern),
|
||||
}
|
||||
|
||||
struct CompiledRule {
|
||||
rule: Rule,
|
||||
conditions: Vec<Check>,
|
||||
exceptions: Vec<Check>,
|
||||
}
|
||||
|
||||
/// The enabled rules, ready to run.
|
||||
pub struct Compiled {
|
||||
rules: Vec<CompiledRule>,
|
||||
}
|
||||
|
||||
/// A rule reference, for notices and the audit record.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct RuleRef {
|
||||
pub id: u32,
|
||||
pub name: String,
|
||||
pub notice: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Match {
|
||||
pub rule_id: u32,
|
||||
pub name: String,
|
||||
pub kind: Kind,
|
||||
pub actions: Vec<Action>,
|
||||
/// Each detector (or `words`, `pattern`) that counted, and its count.
|
||||
pub counts: Vec<(String, usize)>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct Outcome {
|
||||
pub matched: Vec<Match>,
|
||||
pub blocks: Vec<RuleRef>,
|
||||
pub holds: Vec<(RuleRef, bool)>,
|
||||
pub warns: Vec<RuleRef>,
|
||||
}
|
||||
|
||||
/// What DLP decided, strictest first (§2.4).
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Decision {
|
||||
Pass,
|
||||
Block(Vec<RuleRef>),
|
||||
Hold {
|
||||
rules: Vec<RuleRef>,
|
||||
notify_sender: bool,
|
||||
},
|
||||
Warn(Vec<RuleRef>),
|
||||
}
|
||||
|
||||
impl Outcome {
|
||||
/// Block beats hold beats warn. An override (§2.5) answers the warnings
|
||||
/// only: a block or hold still applies.
|
||||
pub fn decision(&self, overridden: bool) -> Decision {
|
||||
if !self.blocks.is_empty() {
|
||||
Decision::Block(self.blocks.clone())
|
||||
} else if !self.holds.is_empty() {
|
||||
Decision::Hold {
|
||||
rules: self.holds.iter().map(|(r, _)| r.clone()).collect(),
|
||||
notify_sender: self.holds.iter().any(|(_, notify)| *notify),
|
||||
}
|
||||
} else if !self.warns.is_empty() && !overridden {
|
||||
Decision::Warn(self.warns.clone())
|
||||
} else {
|
||||
Decision::Pass
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn compile_check(condition: &Condition) -> Result<Check, String> {
|
||||
Ok(match condition {
|
||||
Condition::Words { words, at_least } => Check::Words(WordList::new(words)?, *at_least),
|
||||
Condition::Pattern { pattern, at_least } => {
|
||||
Check::Pattern(Pattern::new(pattern)?, *at_least)
|
||||
}
|
||||
Condition::Header {
|
||||
name,
|
||||
contains,
|
||||
matches,
|
||||
} => Check::Header {
|
||||
name: name.to_ascii_lowercase(),
|
||||
contains: contains.as_ref().map(|c| c.to_lowercase()),
|
||||
matches: matches.as_deref().map(Pattern::new).transpose()?,
|
||||
},
|
||||
Condition::AttachmentName { pattern } => Check::AttachmentName(Pattern::new(pattern)?),
|
||||
other => Check::Plain(other.clone()),
|
||||
})
|
||||
}
|
||||
|
||||
impl Compiled {
|
||||
/// Compiles the enabled rules; one that no longer compiles (a detector
|
||||
/// renamed since it was saved) is skipped and named in the second list.
|
||||
pub fn new(rules: &[Rule]) -> (Self, Vec<(u32, String)>) {
|
||||
let mut compiled = Vec::new();
|
||||
let mut skipped = Vec::new();
|
||||
for rule in rules.iter().filter(|r| r.enabled) {
|
||||
let result = rule.validate().map_err(|e| e.reason).and_then(|_| {
|
||||
Ok(CompiledRule {
|
||||
rule: rule.clone(),
|
||||
conditions: rule
|
||||
.conditions
|
||||
.iter()
|
||||
.map(compile_check)
|
||||
.collect::<Result<_, _>>()?,
|
||||
exceptions: rule
|
||||
.exceptions
|
||||
.iter()
|
||||
.map(compile_check)
|
||||
.collect::<Result<_, _>>()?,
|
||||
})
|
||||
});
|
||||
match result {
|
||||
Ok(c) => compiled.push(c),
|
||||
Err(reason) => skipped.push((rule.id, reason)),
|
||||
}
|
||||
}
|
||||
compiled.sort_by_key(|c| (c.rule.priority, c.rule.id));
|
||||
(Self { rules: compiled }, skipped)
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.rules.is_empty()
|
||||
}
|
||||
|
||||
/// Whether any rule could apply to mail going this way, so a caller can
|
||||
/// skip parsing when none can.
|
||||
pub fn applies_to(&self, outgoing: bool) -> bool {
|
||||
self.rules
|
||||
.iter()
|
||||
.any(|c| direction_matches(c.rule.direction, outgoing))
|
||||
}
|
||||
|
||||
pub fn evaluate(&self, envelope: &Envelope<'_>, content: &Content<'_>) -> Outcome {
|
||||
let mut state = State {
|
||||
content,
|
||||
detected: AHashMap::new(),
|
||||
};
|
||||
let mut outcome = Outcome::default();
|
||||
for compiled in &self.rules {
|
||||
let rule = &compiled.rule;
|
||||
if !direction_matches(rule.direction, envelope.outgoing) {
|
||||
continue;
|
||||
}
|
||||
let mut counts = Vec::new();
|
||||
let all_match = compiled
|
||||
.conditions
|
||||
.iter()
|
||||
.all(|check| state.check(check, envelope, &mut counts));
|
||||
if !all_match {
|
||||
continue;
|
||||
}
|
||||
let mut ignored = Vec::new();
|
||||
if compiled
|
||||
.exceptions
|
||||
.iter()
|
||||
.any(|check| state.check(check, envelope, &mut ignored))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
for action in &rule.actions {
|
||||
let reference = |notice: &str| RuleRef {
|
||||
id: rule.id,
|
||||
name: rule.name.clone(),
|
||||
notice: notice.to_string(),
|
||||
};
|
||||
match action {
|
||||
Action::Block { notice } => outcome.blocks.push(reference(notice)),
|
||||
Action::Hold {
|
||||
notice,
|
||||
notify_sender,
|
||||
} => outcome.holds.push((reference(notice), *notify_sender)),
|
||||
Action::Warn { notice } => outcome.warns.push(reference(notice)),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
outcome.matched.push(Match {
|
||||
rule_id: rule.id,
|
||||
name: rule.name.clone(),
|
||||
kind: rule.kind,
|
||||
actions: rule.actions.clone(),
|
||||
counts,
|
||||
});
|
||||
if rule.stop_processing {
|
||||
break;
|
||||
}
|
||||
}
|
||||
outcome
|
||||
}
|
||||
}
|
||||
|
||||
fn direction_matches(direction: Direction, outgoing: bool) -> bool {
|
||||
match direction {
|
||||
Direction::Any => true,
|
||||
Direction::Outgoing => outgoing,
|
||||
Direction::Incoming => !outgoing,
|
||||
}
|
||||
}
|
||||
|
||||
fn domain_of(address: &str) -> &str {
|
||||
address.rsplit_once('@').map_or("", |(_, d)| d)
|
||||
}
|
||||
|
||||
fn in_list(value: &str, list: &[String]) -> bool {
|
||||
list.iter().any(|v| v.eq_ignore_ascii_case(value))
|
||||
}
|
||||
|
||||
struct State<'c, 'a> {
|
||||
content: &'c Content<'a>,
|
||||
/// Each detector's count, run once per message.
|
||||
detected: AHashMap<&'static str, usize>,
|
||||
}
|
||||
|
||||
impl State<'_, '_> {
|
||||
fn detector_count(&mut self, id: &str) -> usize {
|
||||
let Some(detector) = detectors::by_id(id) else {
|
||||
return 0;
|
||||
};
|
||||
if let Some(count) = self.detected.get(detector.id) {
|
||||
return *count;
|
||||
}
|
||||
let mut findings = Findings::default();
|
||||
for text in self.content.texts() {
|
||||
detector.find(text, &mut findings);
|
||||
}
|
||||
self.detected.insert(detector.id, findings.len());
|
||||
findings.len()
|
||||
}
|
||||
|
||||
fn check(
|
||||
&mut self,
|
||||
check: &Check,
|
||||
envelope: &Envelope<'_>,
|
||||
counts: &mut Vec<(String, usize)>,
|
||||
) -> bool {
|
||||
let content = self.content;
|
||||
match check {
|
||||
Check::Words(list, at_least) => {
|
||||
let n: usize = content.texts().map(|t| list.count(t)).sum();
|
||||
counts.push(("words".into(), n));
|
||||
n >= *at_least as usize
|
||||
}
|
||||
Check::Pattern(pattern, at_least) => {
|
||||
let n: usize = content.texts().map(|t| pattern.count(t)).sum();
|
||||
counts.push(("pattern".into(), n));
|
||||
n >= *at_least as usize
|
||||
}
|
||||
Check::Header {
|
||||
name,
|
||||
contains,
|
||||
matches,
|
||||
} => content
|
||||
.headers
|
||||
.iter()
|
||||
.filter(|(n, _)| n.eq_ignore_ascii_case(name))
|
||||
.any(|(_, value)| match (contains, matches) {
|
||||
(Some(needle), _) => value.to_lowercase().contains(needle.as_str()),
|
||||
(_, Some(pattern)) => pattern.count(value) > 0,
|
||||
_ => true,
|
||||
}),
|
||||
Check::AttachmentName(pattern) => content
|
||||
.attachments
|
||||
.iter()
|
||||
.any(|a| a.name.is_some_and(|n| pattern.count(n) > 0)),
|
||||
Check::Plain(condition) => match condition {
|
||||
Condition::SenderAddress { addresses } => in_list(envelope.sender, addresses),
|
||||
Condition::SenderDomain { domains } => in_list(domain_of(envelope.sender), domains),
|
||||
Condition::SenderGroup { groups } => {
|
||||
envelope.sender_groups.iter().any(|g| groups.contains(g))
|
||||
}
|
||||
Condition::SenderTenant { tenants } => {
|
||||
envelope.sender_tenant.is_some_and(|t| tenants.contains(&t))
|
||||
}
|
||||
Condition::RecipientAddress { addresses } => envelope
|
||||
.recipients
|
||||
.iter()
|
||||
.any(|r| in_list(r.address, addresses)),
|
||||
Condition::RecipientDomain { domains } => envelope
|
||||
.recipients
|
||||
.iter()
|
||||
.any(|r| in_list(domain_of(r.address), domains)),
|
||||
Condition::RecipientGroup { groups } => envelope
|
||||
.recipients
|
||||
.iter()
|
||||
.any(|r| r.groups.iter().any(|g| groups.contains(g))),
|
||||
Condition::RecipientOutside => envelope.recipients.iter().any(|r| !r.local),
|
||||
Condition::AttachmentType { types } => content.attachments.iter().any(|a| {
|
||||
let ct = a.content_type.to_ascii_lowercase();
|
||||
types
|
||||
.iter()
|
||||
.any(|t| ct.starts_with(&t.to_ascii_lowercase()))
|
||||
}),
|
||||
Condition::AttachmentExtension { extensions } => {
|
||||
content.attachments.iter().any(|a| {
|
||||
a.name
|
||||
.and_then(|n| n.rsplit_once('.'))
|
||||
.is_some_and(|(_, ext)| {
|
||||
extensions
|
||||
.iter()
|
||||
.any(|e| e.trim_start_matches('.').eq_ignore_ascii_case(ext))
|
||||
})
|
||||
})
|
||||
}
|
||||
Condition::AttachmentSizeOver { bytes } => {
|
||||
content.attachments.iter().any(|a| a.size > *bytes)
|
||||
}
|
||||
Condition::AttachmentCountOver { count } => {
|
||||
content.attachments.len() > *count as usize
|
||||
}
|
||||
Condition::CantBeInspected => content.cant_be_inspected(),
|
||||
Condition::MessageSizeOver { bytes } => content.size > *bytes,
|
||||
Condition::Detected { detectors } => {
|
||||
let mut any = false;
|
||||
for d in detectors {
|
||||
let n = self.detector_count(&d.id);
|
||||
counts.push((d.id.clone(), n));
|
||||
any |= n >= d.at_least as usize;
|
||||
}
|
||||
any
|
||||
}
|
||||
// Compiled into their own checks
|
||||
Condition::Words { .. }
|
||||
| Condition::Pattern { .. }
|
||||
| Condition::Header { .. }
|
||||
| Condition::AttachmentName { .. } => false,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::mailflow::{
|
||||
extract::Why,
|
||||
rules::{DetectorMin, Position},
|
||||
};
|
||||
|
||||
fn rule(id: u32, kind: Kind, conditions: Vec<Condition>, action: Action) -> Rule {
|
||||
Rule {
|
||||
id,
|
||||
name: format!("rule {id}"),
|
||||
description: String::new(),
|
||||
kind,
|
||||
enabled: true,
|
||||
priority: id as i32,
|
||||
direction: if kind == Kind::Dlp {
|
||||
Direction::Outgoing
|
||||
} else {
|
||||
Direction::Any
|
||||
},
|
||||
conditions,
|
||||
exceptions: vec![],
|
||||
actions: vec![action],
|
||||
stop_processing: false,
|
||||
created_by: String::new(),
|
||||
created_at: 0,
|
||||
updated_at: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn envelope(outside: bool) -> Envelope<'static> {
|
||||
Envelope {
|
||||
outgoing: true,
|
||||
sender: "[email protected]",
|
||||
sender_groups: &[7],
|
||||
sender_tenant: None,
|
||||
recipients: vec![Recipient {
|
||||
address: if outside {
|
||||
"[email protected]"
|
||||
} else {
|
||||
"[email protected]"
|
||||
},
|
||||
local: !outside,
|
||||
groups: &[],
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
fn cards(n: usize) -> Content<'static> {
|
||||
let body: String = [
|
||||
"4242 4242 4242 4242",
|
||||
"5555-5555-5555-4444",
|
||||
"378282246310005",
|
||||
"6011111111111117",
|
||||
"3566002020360505",
|
||||
]
|
||||
.iter()
|
||||
.take(n)
|
||||
.map(|c| format!("card {c}\n"))
|
||||
.collect();
|
||||
Content {
|
||||
subject: "Numbers",
|
||||
bodies: vec![body.into()],
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn five_cards_outside(action: Action) -> Rule {
|
||||
rule(
|
||||
1,
|
||||
Kind::Dlp,
|
||||
vec![
|
||||
Condition::RecipientOutside,
|
||||
Condition::Detected {
|
||||
detectors: vec![DetectorMin {
|
||||
id: "payment-card".into(),
|
||||
at_least: 5,
|
||||
}],
|
||||
},
|
||||
],
|
||||
action,
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detector_threshold_and_recipients() {
|
||||
let (rules, skipped) = Compiled::new(&[five_cards_outside(Action::Hold {
|
||||
notice: "Held".into(),
|
||||
notify_sender: true,
|
||||
})]);
|
||||
assert!(skipped.is_empty());
|
||||
let outcome = rules.evaluate(&envelope(true), &cards(5));
|
||||
assert_eq!(
|
||||
outcome.matched[0].counts,
|
||||
vec![("payment-card".to_string(), 5)]
|
||||
);
|
||||
assert!(matches!(
|
||||
outcome.decision(false),
|
||||
Decision::Hold {
|
||||
notify_sender: true,
|
||||
..
|
||||
}
|
||||
));
|
||||
// Four cards, or everyone inside: nothing
|
||||
assert_eq!(
|
||||
rules.evaluate(&envelope(true), &cards(4)).decision(false),
|
||||
Decision::Pass
|
||||
);
|
||||
assert_eq!(
|
||||
rules.evaluate(&envelope(false), &cards(5)).decision(false),
|
||||
Decision::Pass
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strictest_wins_and_override_answers_warnings_only() {
|
||||
let warn = five_cards_outside(Action::Warn {
|
||||
notice: "Sure?".into(),
|
||||
});
|
||||
let mut block = five_cards_outside(Action::Block {
|
||||
notice: "No".into(),
|
||||
});
|
||||
block.id = 2;
|
||||
let (rules, _) = Compiled::new(&[warn.clone(), block]);
|
||||
let outcome = rules.evaluate(&envelope(true), &cards(5));
|
||||
assert!(matches!(outcome.decision(true), Decision::Block(_)));
|
||||
let (rules, _) = Compiled::new(&[warn]);
|
||||
let outcome = rules.evaluate(&envelope(true), &cards(5));
|
||||
assert!(matches!(outcome.decision(false), Decision::Warn(ref w) if w[0].notice == "Sure?"));
|
||||
assert_eq!(outcome.decision(true), Decision::Pass);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exceptions_order_and_stop_processing() {
|
||||
let disclaimer = |id| {
|
||||
rule(
|
||||
id,
|
||||
Kind::Transport,
|
||||
vec![Condition::RecipientOutside],
|
||||
Action::AddDisclaimer {
|
||||
text: "t".into(),
|
||||
html: None,
|
||||
position: Position::Bottom,
|
||||
},
|
||||
)
|
||||
};
|
||||
let mut first = disclaimer(1);
|
||||
first.stop_processing = true;
|
||||
let (rules, _) = Compiled::new(&[disclaimer(2), first.clone()]);
|
||||
let outcome = rules.evaluate(&envelope(true), &cards(0));
|
||||
assert_eq!(
|
||||
outcome
|
||||
.matched
|
||||
.iter()
|
||||
.map(|m| m.rule_id)
|
||||
.collect::<Vec<_>>(),
|
||||
vec![1]
|
||||
);
|
||||
|
||||
first.stop_processing = false;
|
||||
first.exceptions = vec![Condition::SenderGroup { groups: vec![7] }];
|
||||
let (rules, _) = Compiled::new(&[disclaimer(2), first]);
|
||||
let outcome = rules.evaluate(&envelope(true), &cards(0));
|
||||
assert_eq!(
|
||||
outcome
|
||||
.matched
|
||||
.iter()
|
||||
.map(|m| m.rule_id)
|
||||
.collect::<Vec<_>>(),
|
||||
vec![2]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn content_conditions() {
|
||||
let content = Content {
|
||||
subject: "Project Falcon",
|
||||
bodies: vec!["see attached".into()],
|
||||
headers: vec![("X-Class", "Internal only")],
|
||||
attachments: vec![
|
||||
Attachment {
|
||||
name: Some("plan.docx"),
|
||||
content_type:
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document"
|
||||
.into(),
|
||||
size: 40_000,
|
||||
extracted: Extracted::Text("IBAN GB29 NWBK 6016 1331 9268 19".into()),
|
||||
},
|
||||
Attachment {
|
||||
name: Some("scan.pdf"),
|
||||
content_type: "application/pdf".into(),
|
||||
size: 900_000,
|
||||
extracted: Extracted::NotInspectable(Why::Pdf),
|
||||
},
|
||||
],
|
||||
size: 1_000_000,
|
||||
truncated: false,
|
||||
};
|
||||
let block = || Action::Block { notice: "n".into() };
|
||||
let checks = [
|
||||
(
|
||||
Condition::Words {
|
||||
words: vec!["project falcon".into()],
|
||||
at_least: 1,
|
||||
},
|
||||
true,
|
||||
),
|
||||
(
|
||||
Condition::Header {
|
||||
name: "x-class".into(),
|
||||
contains: Some("internal".into()),
|
||||
matches: None,
|
||||
},
|
||||
true,
|
||||
),
|
||||
(
|
||||
Condition::AttachmentExtension {
|
||||
extensions: vec![".PDF".into()],
|
||||
},
|
||||
true,
|
||||
),
|
||||
(
|
||||
Condition::AttachmentType {
|
||||
types: vec!["image/".into()],
|
||||
},
|
||||
false,
|
||||
),
|
||||
(Condition::AttachmentSizeOver { bytes: 500_000 }, true),
|
||||
(Condition::AttachmentCountOver { count: 2 }, false),
|
||||
(Condition::CantBeInspected, true),
|
||||
(Condition::MessageSizeOver { bytes: 2_000_000 }, false),
|
||||
(
|
||||
Condition::Detected {
|
||||
detectors: vec![DetectorMin {
|
||||
id: "iban".into(),
|
||||
at_least: 1,
|
||||
}],
|
||||
},
|
||||
true,
|
||||
),
|
||||
(
|
||||
Condition::SenderDomain {
|
||||
domains: vec!["EXAMPLE.com".into()],
|
||||
},
|
||||
true,
|
||||
),
|
||||
];
|
||||
for (condition, expected) in checks {
|
||||
let (rules, skipped) =
|
||||
Compiled::new(&[rule(1, Kind::Dlp, vec![condition.clone()], block())]);
|
||||
assert!(skipped.is_empty(), "{condition:?}");
|
||||
let matched = !rules.evaluate(&envelope(true), &content).matched.is_empty();
|
||||
assert_eq!(matched, expected, "{condition:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn direction_and_disabled_rules() {
|
||||
let mut r = five_cards_outside(Action::Block { notice: "n".into() });
|
||||
let (rules, _) = Compiled::new(std::slice::from_ref(&r));
|
||||
assert!(rules.applies_to(true) && !rules.applies_to(false));
|
||||
let mut incoming = envelope(true);
|
||||
incoming.outgoing = false;
|
||||
assert_eq!(
|
||||
rules.evaluate(&incoming, &cards(5)).decision(false),
|
||||
Decision::Pass
|
||||
);
|
||||
r.enabled = false;
|
||||
assert!(Compiled::new(&[r]).0.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,694 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! The text of an attachment, for the detectors (§2.3), or why there isn't
|
||||
//! one.
|
||||
//!
|
||||
//! Read: text files (plain, CSV, JSON, XML, HTML), Office Open XML (DOCX,
|
||||
//! XLSX, PPTX) and OpenDocument (ODT, ODS, ODP) documents, and ZIP archives
|
||||
//! one level deep. **Can't be inspected**: encrypted or password-protected
|
||||
//! files, PDF (settled answer 2), the older binary Office formats, archives
|
||||
//! inside archives, and anything past the limits. Everything else (images,
|
||||
//! audio, programs) has no text to read and is neither.
|
||||
//!
|
||||
//! Office files are ZIP archives of XML, read here with the `zip` and
|
||||
//! `quick-xml` crates the server already uses: no outside converter runs.
|
||||
|
||||
use quick_xml::{Reader, XmlVersion, events::Event};
|
||||
use std::io::{Cursor, Read};
|
||||
|
||||
/// How much may be unpacked from one attachment, and from how many entries.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct Limits {
|
||||
pub max_unpacked: u64,
|
||||
pub max_entries: usize,
|
||||
}
|
||||
|
||||
impl Default for Limits {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_unpacked: 50 * 1024 * 1024,
|
||||
max_entries: 10_000,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Extracted {
|
||||
/// The text to check.
|
||||
Text(String),
|
||||
/// A kind of file with no text in it: nothing to check, nothing missed.
|
||||
NoText,
|
||||
/// A file that may hold text the detectors couldn't read.
|
||||
NotInspectable(Why),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Why {
|
||||
Encrypted,
|
||||
Pdf,
|
||||
LegacyOffice,
|
||||
NestedArchive,
|
||||
TooLarge,
|
||||
Damaged,
|
||||
}
|
||||
|
||||
impl Why {
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Why::Encrypted => "encrypted",
|
||||
Why::Pdf => "pdf",
|
||||
Why::LegacyOffice => "legacy-office",
|
||||
Why::NestedArchive => "nested-archive",
|
||||
Why::TooLarge => "too-large",
|
||||
Why::Damaged => "damaged",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const OLE_MAGIC: &[u8] = &[0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1];
|
||||
const ZIP_MAGIC: &[u8] = b"PK\x03\x04";
|
||||
|
||||
/// What an attachment says, from its declared type, its file name and, above
|
||||
/// all, its first bytes.
|
||||
pub fn extract(
|
||||
content_type: &str,
|
||||
file_name: Option<&str>,
|
||||
data: &[u8],
|
||||
limits: &Limits,
|
||||
) -> Extracted {
|
||||
extract_at(content_type, file_name, data, limits, 0)
|
||||
}
|
||||
|
||||
fn extract_at(
|
||||
content_type: &str,
|
||||
file_name: Option<&str>,
|
||||
data: &[u8],
|
||||
limits: &Limits,
|
||||
depth: u8,
|
||||
) -> Extracted {
|
||||
let content_type = content_type.to_ascii_lowercase();
|
||||
let extension = file_name
|
||||
.and_then(|name| name.rsplit_once('.'))
|
||||
.map(|(_, ext)| ext.to_ascii_lowercase())
|
||||
.unwrap_or_default();
|
||||
|
||||
if data.len() as u64 > limits.max_unpacked {
|
||||
return Extracted::NotInspectable(Why::TooLarge);
|
||||
}
|
||||
if data.starts_with(b"%PDF-") || content_type == "application/pdf" || extension == "pdf" {
|
||||
return Extracted::NotInspectable(Why::Pdf);
|
||||
}
|
||||
if data.starts_with(OLE_MAGIC) {
|
||||
// An encrypted OOXML file is an OLE container holding the encrypted
|
||||
// package; any other OLE file is a legacy .doc, .xls or .ppt
|
||||
return Extracted::NotInspectable(if has_utf16(data, "EncryptedPackage") {
|
||||
Why::Encrypted
|
||||
} else {
|
||||
Why::LegacyOffice
|
||||
});
|
||||
}
|
||||
if data.starts_with(ZIP_MAGIC) {
|
||||
if depth > 0 {
|
||||
return Extracted::NotInspectable(Why::NestedArchive);
|
||||
}
|
||||
return zip(data, limits);
|
||||
}
|
||||
if is_text(&content_type, &extension) {
|
||||
let text = decode_text(data);
|
||||
return Extracted::Text(
|
||||
if content_type == "text/html" || matches!(extension.as_str(), "html" | "htm") {
|
||||
strip_html(&text)
|
||||
} else {
|
||||
text
|
||||
},
|
||||
);
|
||||
}
|
||||
Extracted::NoText
|
||||
}
|
||||
|
||||
fn is_text(content_type: &str, extension: &str) -> bool {
|
||||
content_type.starts_with("text/")
|
||||
|| matches!(
|
||||
content_type,
|
||||
"application/json"
|
||||
| "application/xml"
|
||||
| "application/csv"
|
||||
| "application/x-csv"
|
||||
| "message/rfc822"
|
||||
)
|
||||
|| matches!(
|
||||
extension,
|
||||
"txt"
|
||||
| "csv"
|
||||
| "tsv"
|
||||
| "json"
|
||||
| "xml"
|
||||
| "md"
|
||||
| "log"
|
||||
| "html"
|
||||
| "htm"
|
||||
| "eml"
|
||||
| "ics"
|
||||
| "vcf"
|
||||
)
|
||||
}
|
||||
|
||||
/// UTF-16 with a byte order mark, else UTF-8 (lossy).
|
||||
fn decode_text(data: &[u8]) -> String {
|
||||
let utf16 = |bytes: &[u8], big: bool| {
|
||||
let units: Vec<u16> = bytes
|
||||
.as_chunks::<2>()
|
||||
.0
|
||||
.iter()
|
||||
.map(|&c| {
|
||||
if big {
|
||||
u16::from_be_bytes(c)
|
||||
} else {
|
||||
u16::from_le_bytes(c)
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
String::from_utf16_lossy(&units)
|
||||
};
|
||||
match data {
|
||||
[0xFF, 0xFE, rest @ ..] => utf16(rest, false),
|
||||
[0xFE, 0xFF, rest @ ..] => utf16(rest, true),
|
||||
[0xEF, 0xBB, 0xBF, rest @ ..] => String::from_utf8_lossy(rest).into_owned(),
|
||||
_ => String::from_utf8_lossy(data).into_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
fn has_utf16(data: &[u8], needle: &str) -> bool {
|
||||
let needle: Vec<u8> = needle.encode_utf16().flat_map(u16::to_le_bytes).collect();
|
||||
data.windows(needle.len()).any(|w| w == needle.as_slice())
|
||||
}
|
||||
|
||||
/// Tags out, the common entities decoded, block ends as new lines.
|
||||
fn strip_html(html: &str) -> String {
|
||||
let mut out = String::with_capacity(html.len());
|
||||
let mut in_tag = false;
|
||||
let mut skip_until: Option<&str> = None;
|
||||
let lower = html.to_ascii_lowercase();
|
||||
let mut i = 0;
|
||||
let bytes = html.as_bytes();
|
||||
while i < bytes.len() {
|
||||
if let Some(end) = skip_until {
|
||||
match lower[i..].find(end) {
|
||||
Some(at) => {
|
||||
i += at + end.len();
|
||||
skip_until = None;
|
||||
}
|
||||
None => break,
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let c = bytes[i];
|
||||
if in_tag {
|
||||
if c == b'>' {
|
||||
in_tag = false;
|
||||
}
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if c == b'<' {
|
||||
if lower[i..].starts_with("<script") {
|
||||
skip_until = Some("</script>");
|
||||
} else if lower[i..].starts_with("<style") {
|
||||
skip_until = Some("</style>");
|
||||
} else {
|
||||
if [
|
||||
"<br", "<p", "</p", "<div", "</div", "<tr", "<li", "<td", "<th",
|
||||
]
|
||||
.iter()
|
||||
.any(|t| lower[i..].starts_with(t))
|
||||
{
|
||||
out.push(
|
||||
if lower[i..].starts_with("<td") || lower[i..].starts_with("<th") {
|
||||
'\t'
|
||||
} else {
|
||||
'\n'
|
||||
},
|
||||
);
|
||||
}
|
||||
in_tag = true;
|
||||
}
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
// Copy up to the next tag
|
||||
let next = html[i..].find('<').map_or(html.len(), |at| i + at);
|
||||
out.push_str(&html[i..next]);
|
||||
i = next;
|
||||
}
|
||||
for (entity, text) in [
|
||||
(" ", " "),
|
||||
("<", "<"),
|
||||
(">", ">"),
|
||||
(""", "\""),
|
||||
("'", "'"),
|
||||
("&", "&"),
|
||||
] {
|
||||
out = out.replace(entity, text);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// A ZIP file: an Office document, an OpenDocument, or an archive.
|
||||
fn zip(data: &[u8], limits: &Limits) -> Extracted {
|
||||
let Ok(mut archive) = zip::ZipArchive::new(Cursor::new(data)) else {
|
||||
return Extracted::NotInspectable(Why::Damaged);
|
||||
};
|
||||
if archive.len() > limits.max_entries {
|
||||
return Extracted::NotInspectable(Why::TooLarge);
|
||||
}
|
||||
let mut names = Vec::with_capacity(archive.len());
|
||||
let mut declared: u64 = 0;
|
||||
for i in 0..archive.len() {
|
||||
let Ok(entry) = archive.by_index_raw(i) else {
|
||||
return Extracted::NotInspectable(Why::Damaged);
|
||||
};
|
||||
if entry.encrypted() {
|
||||
return Extracted::NotInspectable(Why::Encrypted);
|
||||
}
|
||||
declared = declared.saturating_add(entry.size());
|
||||
names.push(entry.name().to_string());
|
||||
}
|
||||
if declared > limits.max_unpacked {
|
||||
return Extracted::NotInspectable(Why::TooLarge);
|
||||
}
|
||||
let mut budget = limits.max_unpacked;
|
||||
let mut read =
|
||||
|archive: &mut zip::ZipArchive<Cursor<&[u8]>>, name: &str| -> Result<Vec<u8>, Why> {
|
||||
let entry = archive.by_name(name).map_err(|_| Why::Damaged)?;
|
||||
let mut bytes = Vec::new();
|
||||
// Declared sizes can lie: stop at the budget whatever they say
|
||||
entry
|
||||
.take(budget + 1)
|
||||
.read_to_end(&mut bytes)
|
||||
.map_err(|_| Why::Damaged)?;
|
||||
if bytes.len() as u64 > budget {
|
||||
return Err(Why::TooLarge);
|
||||
}
|
||||
budget -= bytes.len() as u64;
|
||||
Ok(bytes)
|
||||
};
|
||||
|
||||
let has = |name: &str| names.iter().any(|n| n == name);
|
||||
let mut text = String::new();
|
||||
let result: Result<(), Why> = (|| {
|
||||
if has("[Content_Types].xml") {
|
||||
// Office Open XML: the parts that hold what a person wrote
|
||||
let mut shared = Vec::new();
|
||||
if has("xl/sharedStrings.xml") {
|
||||
shared = xml_strings(&read(&mut archive, "xl/sharedStrings.xml")?, "si");
|
||||
}
|
||||
for name in names.iter().filter(|n| ooxml_text_part(n)) {
|
||||
let xml = read(&mut archive, name)?;
|
||||
if name.starts_with("xl/worksheets/") {
|
||||
xlsx_sheet(&xml, &mut text);
|
||||
} else {
|
||||
xml_text(&xml, &mut text);
|
||||
}
|
||||
text.push('\n');
|
||||
}
|
||||
text.extend(shared.iter().map(|s| format!("{s}\n")));
|
||||
} else if names.first().is_some_and(|n| n == "mimetype")
|
||||
&& read(&mut archive, "mimetype")?.starts_with(b"application/vnd.oasis.opendocument")
|
||||
{
|
||||
// OpenDocument: an encrypted one says so in its manifest
|
||||
if has("META-INF/manifest.xml")
|
||||
&& contains(
|
||||
&read(&mut archive, "META-INF/manifest.xml")?,
|
||||
b"encryption-data",
|
||||
)
|
||||
{
|
||||
return Err(Why::Encrypted);
|
||||
}
|
||||
for name in ["content.xml", "styles.xml"] {
|
||||
if has(name) {
|
||||
xml_text(&read(&mut archive, name)?, &mut text);
|
||||
text.push('\n');
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// An archive: each file inside, one level deep
|
||||
for name in names.iter().filter(|n| !n.ends_with('/')) {
|
||||
let bytes = read(&mut archive, name)?;
|
||||
match extract_at("", Some(name), &bytes, limits, 1) {
|
||||
Extracted::Text(inner) => {
|
||||
text.push_str(&inner);
|
||||
text.push('\n');
|
||||
}
|
||||
Extracted::NoText => {}
|
||||
Extracted::NotInspectable(why) => return Err(why),
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
match result {
|
||||
Ok(()) => Extracted::Text(text),
|
||||
Err(why) => Extracted::NotInspectable(why),
|
||||
}
|
||||
}
|
||||
|
||||
fn ooxml_text_part(name: &str) -> bool {
|
||||
let xml = name.ends_with(".xml");
|
||||
xml && (name == "word/document.xml"
|
||||
|| [
|
||||
"word/header",
|
||||
"word/footer",
|
||||
"word/footnotes",
|
||||
"word/endnotes",
|
||||
"word/comments",
|
||||
]
|
||||
.iter()
|
||||
.any(|p| name.starts_with(p))
|
||||
|| name.starts_with("xl/worksheets/sheet")
|
||||
|| name.starts_with("ppt/slides/slide")
|
||||
|| name.starts_with("ppt/notesSlides/"))
|
||||
}
|
||||
|
||||
fn contains(haystack: &[u8], needle: &[u8]) -> bool {
|
||||
haystack.windows(needle.len()).any(|w| w == needle)
|
||||
}
|
||||
|
||||
/// The local name of a tag, without its namespace prefix.
|
||||
fn local(name: &[u8]) -> &[u8] {
|
||||
name.rsplit(|b| *b == b':').next().unwrap_or(name)
|
||||
}
|
||||
|
||||
fn push_entity(entity: &[u8], out: &mut String) {
|
||||
match entity {
|
||||
b"lt" => out.push('<'),
|
||||
b"gt" => out.push('>'),
|
||||
b"amp" => out.push('&'),
|
||||
b"apos" => out.push('\''),
|
||||
b"quot" => out.push('"'),
|
||||
_ => {
|
||||
let code = match entity {
|
||||
[b'#', b'x' | b'X', hex @ ..] => std::str::from_utf8(hex)
|
||||
.ok()
|
||||
.and_then(|h| u32::from_str_radix(h, 16).ok()),
|
||||
[b'#', dec @ ..] => std::str::from_utf8(dec).ok().and_then(|d| d.parse().ok()),
|
||||
_ => None,
|
||||
};
|
||||
if let Some(c) = code.and_then(char::from_u32) {
|
||||
out.push(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Every text node, runs joined as written, a new line after each paragraph
|
||||
/// or row and a tab after each cell, so a number split across runs is whole
|
||||
/// again.
|
||||
fn xml_text(xml: &[u8], out: &mut String) {
|
||||
let mut reader = Reader::from_reader(xml);
|
||||
let mut buf = Vec::new();
|
||||
loop {
|
||||
match reader.read_event_into(&mut buf) {
|
||||
Ok(Event::Text(t)) => {
|
||||
if let Ok(text) = t.xml_content(XmlVersion::Implicit1_0) {
|
||||
out.push_str(&text);
|
||||
}
|
||||
}
|
||||
Ok(Event::CData(t)) => out.push_str(&String::from_utf8_lossy(&t)),
|
||||
Ok(Event::GeneralRef(entity)) => push_entity(&entity, out),
|
||||
Ok(Event::End(e)) => match local(e.name().as_ref()) {
|
||||
b"p" | b"h" | b"tr" | b"row" | b"table-row" | b"br" => out.push('\n'),
|
||||
b"tc" | b"c" | b"table-cell" | b"tab" => out.push('\t'),
|
||||
_ => {}
|
||||
},
|
||||
Ok(Event::Empty(e)) => match local(e.name().as_ref()) {
|
||||
b"br" | b"line-break" => out.push('\n'),
|
||||
b"tab" | b"s" => out.push(' '),
|
||||
_ => {}
|
||||
},
|
||||
Ok(Event::Eof) | Err(_) => break,
|
||||
_ => {}
|
||||
}
|
||||
buf.clear();
|
||||
}
|
||||
}
|
||||
|
||||
/// The text of each `item` element (a shared string in XLSX).
|
||||
fn xml_strings(xml: &[u8], item: &str) -> Vec<String> {
|
||||
let mut reader = Reader::from_reader(xml);
|
||||
let mut buf = Vec::new();
|
||||
let mut items = Vec::new();
|
||||
let mut current: Option<String> = None;
|
||||
loop {
|
||||
match reader.read_event_into(&mut buf) {
|
||||
Ok(Event::Start(e)) if local(e.name().as_ref()) == item.as_bytes() => {
|
||||
current = Some(String::new())
|
||||
}
|
||||
Ok(Event::End(e)) if local(e.name().as_ref()) == item.as_bytes() => {
|
||||
items.extend(current.take());
|
||||
}
|
||||
Ok(Event::Text(t)) => {
|
||||
if let (Some(s), Ok(text)) =
|
||||
(current.as_mut(), t.xml_content(XmlVersion::Implicit1_0))
|
||||
{
|
||||
s.push_str(&text);
|
||||
}
|
||||
}
|
||||
Ok(Event::GeneralRef(entity)) => {
|
||||
if let Some(s) = current.as_mut() {
|
||||
push_entity(&entity, s);
|
||||
}
|
||||
}
|
||||
Ok(Event::Eof) | Err(_) => break,
|
||||
_ => {}
|
||||
}
|
||||
buf.clear();
|
||||
}
|
||||
items
|
||||
}
|
||||
|
||||
/// A worksheet's cell values: numbers and inline strings. Cells holding a
|
||||
/// shared string are skipped here; the shared strings are read whole.
|
||||
fn xlsx_sheet(xml: &[u8], out: &mut String) {
|
||||
let mut reader = Reader::from_reader(xml);
|
||||
let mut buf = Vec::new();
|
||||
let mut shared_cell = false;
|
||||
let mut in_value = false;
|
||||
loop {
|
||||
match reader.read_event_into(&mut buf) {
|
||||
Ok(Event::Start(e)) => match local(e.name().as_ref()) {
|
||||
b"c" => {
|
||||
shared_cell = e
|
||||
.attributes()
|
||||
.flatten()
|
||||
.any(|a| a.key.as_ref() == b"t" && a.value.as_ref() == b"s");
|
||||
}
|
||||
b"v" | b"t" => in_value = true,
|
||||
_ => {}
|
||||
},
|
||||
Ok(Event::End(e)) => match local(e.name().as_ref()) {
|
||||
b"v" | b"t" => in_value = false,
|
||||
b"c" => out.push('\t'),
|
||||
b"row" => out.push('\n'),
|
||||
_ => {}
|
||||
},
|
||||
// A shared string's cell holds only its index: the string itself
|
||||
// is added with the shared strings
|
||||
Ok(Event::Text(t)) if in_value && !shared_cell => {
|
||||
if let Ok(text) = t.xml_content(XmlVersion::Implicit1_0) {
|
||||
out.push_str(&text);
|
||||
}
|
||||
}
|
||||
Ok(Event::Eof) | Err(_) => break,
|
||||
_ => {}
|
||||
}
|
||||
buf.clear();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
use zip::{ZipWriter, write::SimpleFileOptions};
|
||||
|
||||
fn zip_of(files: &[(&str, &str)]) -> Vec<u8> {
|
||||
let mut zip = ZipWriter::new(Cursor::new(Vec::new()));
|
||||
for (name, body) in files {
|
||||
zip.start_file(*name, SimpleFileOptions::default()).unwrap();
|
||||
zip.write_all(body.as_bytes()).unwrap();
|
||||
}
|
||||
zip.finish().unwrap().into_inner()
|
||||
}
|
||||
|
||||
fn text_of(extracted: Extracted) -> String {
|
||||
match extracted {
|
||||
Extracted::Text(text) => text,
|
||||
other => panic!("expected text, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plain_text_and_html() {
|
||||
let limits = Limits::default();
|
||||
assert_eq!(
|
||||
text_of(extract("text/plain", None, b"card 4242", &limits)),
|
||||
"card 4242"
|
||||
);
|
||||
let utf16: Vec<u8> = [0xFF, 0xFE]
|
||||
.into_iter()
|
||||
.chain("héllo".encode_utf16().flat_map(u16::to_le_bytes))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
text_of(extract(
|
||||
"application/octet-stream",
|
||||
Some("a.csv"),
|
||||
&utf16,
|
||||
&limits
|
||||
)),
|
||||
"héllo"
|
||||
);
|
||||
let html = "<html><style>p{}</style><p>Card 4242</p><script>x()</script><td>a</td><td>b</td></html>";
|
||||
let text = text_of(extract("text/html", None, html.as_bytes(), &limits));
|
||||
assert!(
|
||||
text.contains("Card 4242") && !text.contains("x()") && !text.contains("p{}"),
|
||||
"{text:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
extract("image/png", Some("a.png"), b"\x89PNG....", &limits),
|
||||
Extracted::NoText
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn docx_joins_split_runs() {
|
||||
let doc = r#"<w:document xmlns:w="w"><w:body><w:p><w:r><w:t>Card 4242 42</w:t></w:r><w:r><w:t>42 4242 4242</w:t></w:r></w:p><w:p><w:r><w:t>A & B</w:t></w:r></w:p></w:body></w:document>"#;
|
||||
let docx = zip_of(&[
|
||||
("[Content_Types].xml", "<Types/>"),
|
||||
("word/document.xml", doc),
|
||||
]);
|
||||
let text = text_of(extract(
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
Some("a.docx"),
|
||||
&docx,
|
||||
&Limits::default(),
|
||||
));
|
||||
assert!(text.contains("Card 4242 4242 4242 4242\nA & B"), "{text:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xlsx_numbers_and_shared_strings() {
|
||||
let sheet = r#"<worksheet><sheetData><row><c r="A1" t="s"><v>0</v></c><c r="B1"><v>4242424242424242</v></c></row></sheetData></worksheet>"#;
|
||||
let shared = r#"<sst><si><t>IBAN GB29 NWBK 6016 1331 9268 19</t></si></sst>"#;
|
||||
let xlsx = zip_of(&[
|
||||
("[Content_Types].xml", "<Types/>"),
|
||||
("xl/sharedStrings.xml", shared),
|
||||
("xl/worksheets/sheet1.xml", sheet),
|
||||
]);
|
||||
let text = text_of(extract("", Some("book.xlsx"), &xlsx, &Limits::default()));
|
||||
assert!(
|
||||
text.contains("4242424242424242") && text.contains("GB29 NWBK 6016 1331 9268 19"),
|
||||
"{text:?}"
|
||||
);
|
||||
// The shared string's index isn't read as a value
|
||||
assert!(
|
||||
!text.contains("\t0\t") && !text.starts_with('0'),
|
||||
"{text:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn opendocument_and_encrypted_opendocument() {
|
||||
let content = r#"<office:document-content xmlns:text="t"><text:p>SSN 078-05-1120</text:p></office:document-content>"#;
|
||||
let odt = zip_of(&[
|
||||
("mimetype", "application/vnd.oasis.opendocument.text"),
|
||||
("content.xml", content),
|
||||
]);
|
||||
assert!(
|
||||
text_of(extract("", Some("a.odt"), &odt, &Limits::default()))
|
||||
.contains("SSN 078-05-1120")
|
||||
);
|
||||
let manifest = r#"<manifest:manifest><manifest:file-entry><manifest:encryption-data/></manifest:file-entry></manifest:manifest>"#;
|
||||
let locked = zip_of(&[
|
||||
("mimetype", "application/vnd.oasis.opendocument.text"),
|
||||
("META-INF/manifest.xml", manifest),
|
||||
("content.xml", "x"),
|
||||
]);
|
||||
assert_eq!(
|
||||
extract("", Some("a.odt"), &locked, &Limits::default()),
|
||||
Extracted::NotInspectable(Why::Encrypted)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn archives() {
|
||||
let limits = Limits::default();
|
||||
let archive = zip_of(&[
|
||||
("notes/a.txt", "card 4242424242424242"),
|
||||
("b.png", "\u{89}PNG"),
|
||||
]);
|
||||
assert!(
|
||||
text_of(extract("application/zip", Some("x.zip"), &archive, &limits))
|
||||
.contains("4242424242424242")
|
||||
);
|
||||
let nested = zip_of(&[(
|
||||
"inner.zip",
|
||||
std::str::from_utf8(&[b'P', b'K', 3, 4]).unwrap(),
|
||||
)]);
|
||||
assert_eq!(
|
||||
extract("application/zip", Some("x.zip"), &nested, &limits),
|
||||
Extracted::NotInspectable(Why::NestedArchive)
|
||||
);
|
||||
|
||||
// Password-protected
|
||||
let mut zip = ZipWriter::new(Cursor::new(Vec::new()));
|
||||
zip.start_file(
|
||||
"secret.txt",
|
||||
SimpleFileOptions::default().with_aes_encryption(zip::AesMode::Aes256, "pw"),
|
||||
)
|
||||
.unwrap();
|
||||
zip.write_all(b"4242424242424242").unwrap();
|
||||
let locked = zip.finish().unwrap().into_inner();
|
||||
assert_eq!(
|
||||
extract("application/zip", Some("x.zip"), &locked, &limits),
|
||||
Extracted::NotInspectable(Why::Encrypted)
|
||||
);
|
||||
|
||||
// Past the limits
|
||||
let small = Limits {
|
||||
max_unpacked: 10,
|
||||
max_entries: 1,
|
||||
};
|
||||
assert_eq!(
|
||||
extract("application/zip", Some("x.zip"), &archive, &small),
|
||||
Extracted::NotInspectable(Why::TooLarge)
|
||||
);
|
||||
assert_eq!(
|
||||
extract("application/zip", None, b"PK\x03\x04garbage", &limits),
|
||||
Extracted::NotInspectable(Why::Damaged)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn not_inspectable_kinds() {
|
||||
let limits = Limits::default();
|
||||
assert_eq!(
|
||||
extract("application/octet-stream", None, b"%PDF-1.7 ...", &limits),
|
||||
Extracted::NotInspectable(Why::Pdf)
|
||||
);
|
||||
let mut ole = OLE_MAGIC.to_vec();
|
||||
ole.extend(std::iter::repeat_n(0, 64));
|
||||
assert_eq!(
|
||||
extract("", Some("old.doc"), &ole, &limits),
|
||||
Extracted::NotInspectable(Why::LegacyOffice)
|
||||
);
|
||||
ole.extend("EncryptedPackage".encode_utf16().flat_map(u16::to_le_bytes));
|
||||
assert_eq!(
|
||||
extract("", Some("new.docx"), &ole, &limits),
|
||||
Extracted::NotInspectable(Why::Encrypted)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! Mail held for review (dlp-and-mail-flow-rules spec, §2.6).
|
||||
//!
|
||||
//! A held message is queued as any other, but released [`HOLD_SECONDS`]
|
||||
//! from now, the queue's own future-release mechanism: nothing about the
|
||||
//! queue's stored format changes, so a node on an older version reads it
|
||||
//! and simply never sends it. Beside it, a review record under `R` `h` +
|
||||
//! queue id (u64) says why it's held, for the review queue.
|
||||
//!
|
||||
//! A reviewer releases it (it's rescheduled from the queue's settings and
|
||||
//! delivered) or rejects it (it's removed, and the sender told). Unreviewed
|
||||
//! mail is rejected after [`KEEP_DAYS`].
|
||||
|
||||
use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize, de::DeserializeOwned};
|
||||
use store::{
|
||||
Deserialize, IterateParams, SUBSPACE_INBUXA, Serialize, Store, ValueKey,
|
||||
write::{AnyClass, BatchBuilder, ValueClass},
|
||||
};
|
||||
use trc::AddContext;
|
||||
|
||||
const FEATURE: u8 = b'R';
|
||||
const KIND_HELD: u8 = b'h';
|
||||
const KIND_SETTINGS: u8 = b's';
|
||||
|
||||
/// How far off a held message's release is set: a century, so it never
|
||||
/// comes due on its own.
|
||||
pub const HOLD_SECONDS: u64 = 100 * 365 * 24 * 60 * 60;
|
||||
|
||||
/// How long unreviewed mail waits before it's rejected, unless the setting
|
||||
/// says otherwise (settled answer 5).
|
||||
pub const KEEP_DAYS: u64 = 7;
|
||||
|
||||
/// `inbuxa:DlpSettings`: how many days held mail waits for a reviewer.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Settings {
|
||||
pub keep_held_days: u64,
|
||||
}
|
||||
|
||||
impl Default for Settings {
|
||||
fn default() -> Self {
|
||||
Settings {
|
||||
keep_held_days: KEEP_DAYS,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Settings {
|
||||
/// The property at fault and why, or fine.
|
||||
pub fn check(&self) -> Result<(), (&'static str, &'static str)> {
|
||||
if (1..=90).contains(&self.keep_held_days) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(("keepHeldDays", "must be from 1 to 90 days"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A rule that held the message, with its notice.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
|
||||
pub struct HeldRule {
|
||||
pub name: String,
|
||||
pub notice: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Held {
|
||||
pub queue_id: u64,
|
||||
pub sender: String,
|
||||
#[serde(default)]
|
||||
pub account_id: Option<u32>,
|
||||
#[serde(default)]
|
||||
pub tenant_id: Option<u32>,
|
||||
pub recipients: Vec<String>,
|
||||
pub subject: String,
|
||||
pub size: u64,
|
||||
pub rules: Vec<HeldRule>,
|
||||
/// Each detector that counted, and its count.
|
||||
#[serde(default)]
|
||||
pub counts: Vec<(String, usize)>,
|
||||
/// Seconds since the epoch.
|
||||
pub held_at: u64,
|
||||
pub expires_at: u64,
|
||||
/// The days it was given, for what the sender is told.
|
||||
#[serde(default = "default_keep_days")]
|
||||
pub keep_days: u64,
|
||||
}
|
||||
|
||||
fn default_keep_days() -> u64 {
|
||||
KEEP_DAYS
|
||||
}
|
||||
|
||||
impl Held {
|
||||
pub fn is_expired(&self, now: u64) -> bool {
|
||||
now >= self.expires_at
|
||||
}
|
||||
}
|
||||
|
||||
struct Json<T>(T);
|
||||
|
||||
impl<T: SerdeSerialize> Serialize for Json<T> {
|
||||
fn serialize(&self) -> trc::Result<Vec<u8>> {
|
||||
serde_json::to_vec(&self.0).map_err(|err| {
|
||||
trc::StoreEvent::UnexpectedError
|
||||
.into_err()
|
||||
.details("Failed to serialize held message")
|
||||
.reason(err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: DeserializeOwned + Sync + Send> Deserialize for Json<T> {
|
||||
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
|
||||
serde_json::from_slice(bytes).map(Json).map_err(|err| {
|
||||
trc::StoreEvent::DataCorruption
|
||||
.into_err()
|
||||
.details("Invalid held message")
|
||||
.reason(err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn class(queue_id: u64) -> ValueClass {
|
||||
let mut key = Vec::with_capacity(10);
|
||||
key.push(FEATURE);
|
||||
key.push(KIND_HELD);
|
||||
key.extend_from_slice(&queue_id.to_be_bytes());
|
||||
ValueClass::Any(AnyClass {
|
||||
subspace: SUBSPACE_INBUXA,
|
||||
key,
|
||||
})
|
||||
}
|
||||
|
||||
fn key(queue_id: u64) -> ValueKey<ValueClass> {
|
||||
ValueKey::from(class(queue_id))
|
||||
}
|
||||
|
||||
fn settings_class() -> ValueClass {
|
||||
ValueClass::Any(AnyClass {
|
||||
subspace: SUBSPACE_INBUXA,
|
||||
key: vec![FEATURE, KIND_SETTINGS],
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn settings(data: &Store) -> trc::Result<Settings> {
|
||||
Ok(data
|
||||
.get_value::<Json<Settings>>(ValueKey::from(settings_class()))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.map(|Json(settings)| settings)
|
||||
.unwrap_or_default())
|
||||
}
|
||||
|
||||
pub async fn set_settings(data: &Store, settings: &Settings) -> trc::Result<()> {
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.set(settings_class(), Json(settings).serialize()?);
|
||||
data.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get(data: &Store, queue_id: u64) -> trc::Result<Option<Held>> {
|
||||
Ok(data
|
||||
.get_value::<Json<Held>>(key(queue_id))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.map(|Json(held)| held))
|
||||
}
|
||||
|
||||
pub async fn is_held(data: &Store, queue_id: u64) -> trc::Result<bool> {
|
||||
get(data, queue_id).await.map(|held| held.is_some())
|
||||
}
|
||||
|
||||
/// Every held message, oldest first.
|
||||
pub async fn all(data: &Store) -> trc::Result<Vec<Held>> {
|
||||
let mut held = Vec::new();
|
||||
data.iterate(IterateParams::new(key(0), key(u64::MAX)), |_, value| {
|
||||
if let Ok(Json(record)) = Json::<Held>::deserialize(value) {
|
||||
held.push(record);
|
||||
}
|
||||
Ok(true)
|
||||
})
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
held.sort_by_key(|h| (h.held_at, h.queue_id));
|
||||
Ok(held)
|
||||
}
|
||||
|
||||
pub async fn create(data: &Store, held: &Held) -> trc::Result<()> {
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.set(class(held.queue_id), Json(held).serialize()?);
|
||||
data.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn delete(data: &Store, queue_id: u64) -> trc::Result<()> {
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.clear(class(queue_id));
|
||||
data.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn wire_format_and_expiry() {
|
||||
let held = Held {
|
||||
queue_id: 42,
|
||||
sender: "[email protected]".into(),
|
||||
account_id: Some(7),
|
||||
tenant_id: None,
|
||||
recipients: vec!["[email protected]".into()],
|
||||
subject: "Numbers".into(),
|
||||
size: 900,
|
||||
rules: vec![HeldRule {
|
||||
name: "Cards".into(),
|
||||
notice: "Held for review".into(),
|
||||
}],
|
||||
counts: vec![("payment-card".into(), 5)],
|
||||
held_at: 1_000,
|
||||
expires_at: 1_000 + KEEP_DAYS * 86_400,
|
||||
keep_days: KEEP_DAYS,
|
||||
};
|
||||
let json = serde_json::to_value(&held).unwrap();
|
||||
assert_eq!(json["heldAt"], 1_000);
|
||||
assert_eq!(serde_json::from_value::<Held>(json).unwrap(), held);
|
||||
assert!(!held.is_expired(1_000 + KEEP_DAYS * 86_400 - 1));
|
||||
assert!(held.is_expired(1_000 + KEEP_DAYS * 86_400));
|
||||
assert!(HOLD_SECONDS > 90 * 365 * 86_400);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_range() {
|
||||
assert_eq!(Settings::default().keep_held_days, 7);
|
||||
assert!(Settings { keep_held_days: 1 }.check().is_ok());
|
||||
assert!(Settings { keep_held_days: 90 }.check().is_ok());
|
||||
assert!(Settings { keep_held_days: 0 }.check().is_err());
|
||||
assert!(Settings { keep_held_days: 91 }.check().is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! Data loss prevention and mail flow rules (dlp-and-mail-flow-rules spec).
|
||||
//!
|
||||
//! Mostly pure functions over text and attachment bytes, unit-tested
|
||||
//! without a server:
|
||||
//!
|
||||
//! - [`detectors`]: find identifiers in text (payment cards, IBANs,
|
||||
//! national ID numbers, keys), each by its published format and check
|
||||
//! (§2.3);
|
||||
//! - [`words`]: an organization's own word lists and patterns;
|
||||
//! - [`extract`]: the text of an attachment, or why it can't be read;
|
||||
//! - [`rules`]: what a rule is, its checks, and where rules are kept;
|
||||
//! - [`engine`]: rules compiled and run against a message;
|
||||
//! - [`cache`]: each node's compiled copy;
|
||||
//! - [`rewrite`]: the actions that change a message.
|
||||
//!
|
||||
//! Nothing here writes what it finds anywhere: callers get counts, and the
|
||||
//! matched text never leaves the evaluation (§2.7).
|
||||
|
||||
pub mod cache;
|
||||
pub mod detectors;
|
||||
pub mod engine;
|
||||
pub mod extract;
|
||||
pub mod held;
|
||||
pub mod rewrite;
|
||||
pub mod rules;
|
||||
pub mod words;
|
||||
@@ -0,0 +1,306 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! Transport actions that change a message (§2.4): headers, the subject,
|
||||
//! disclaimers. Each takes the raw message and returns the new one, or
|
||||
//! `None` when there's nothing to change.
|
||||
//!
|
||||
//! Only what the action names changes. A disclaimer edits the message's
|
||||
//! main text and HTML bodies (not attachments, not attached messages):
|
||||
//! each is decoded, changed and written back as UTF-8 quoted-printable,
|
||||
//! with its other headers kept. A disclaimer already there isn't added
|
||||
//! again, so a reply thread carries it once.
|
||||
|
||||
use base64::{Engine, engine::general_purpose::STANDARD};
|
||||
use mail_builder::encoders::quoted_printable::QuotedPrintableEncoder;
|
||||
use mail_parser::{HeaderName, MessageParser, PartType};
|
||||
|
||||
use super::rules::Position;
|
||||
|
||||
/// A header value, as an RFC 2047 encoded word when it isn't plain ASCII.
|
||||
pub fn header_value(value: &str) -> String {
|
||||
if value.is_ascii() {
|
||||
value.to_string()
|
||||
} else {
|
||||
format!("=?utf-8?B?{}?=", STANDARD.encode(value))
|
||||
}
|
||||
}
|
||||
|
||||
/// `Name: value` added at the top of the message.
|
||||
pub fn add_header(message: &[u8], name: &str, value: &str) -> Vec<u8> {
|
||||
let mut out = Vec::with_capacity(message.len() + name.len() + value.len() + 4);
|
||||
out.extend_from_slice(name.as_bytes());
|
||||
out.extend_from_slice(b": ");
|
||||
out.extend_from_slice(header_value(value).as_bytes());
|
||||
out.extend_from_slice(b"\r\n");
|
||||
out.extend_from_slice(message);
|
||||
out
|
||||
}
|
||||
|
||||
/// Every top-level header called `name` taken out.
|
||||
pub fn remove_header(message: &[u8], name: &str) -> Option<Vec<u8>> {
|
||||
let parsed = MessageParser::new().parse_headers(message)?;
|
||||
let mut ranges: Vec<(usize, usize)> = parsed
|
||||
.headers()
|
||||
.iter()
|
||||
.filter(|h| h.name.as_str().eq_ignore_ascii_case(name))
|
||||
.map(|h| (h.offset_field as usize, h.offset_end as usize))
|
||||
.collect();
|
||||
if ranges.is_empty() {
|
||||
return None;
|
||||
}
|
||||
ranges.sort_unstable();
|
||||
let mut out = Vec::with_capacity(message.len());
|
||||
let mut at = 0;
|
||||
for (start, end) in ranges {
|
||||
out.extend_from_slice(&message[at..start]);
|
||||
at = end;
|
||||
}
|
||||
out.extend_from_slice(&message[at..]);
|
||||
Some(out)
|
||||
}
|
||||
|
||||
/// The Subject header replaced by `subject` (added if there was none).
|
||||
pub fn set_subject(message: &[u8], subject: &str) -> Vec<u8> {
|
||||
let line = format!("Subject: {}\r\n", header_value(subject));
|
||||
let parsed = MessageParser::new().parse_headers(message);
|
||||
match parsed
|
||||
.as_ref()
|
||||
.and_then(|p| p.headers().iter().find(|h| h.name == HeaderName::Subject))
|
||||
{
|
||||
Some(header) => {
|
||||
let mut out = Vec::with_capacity(message.len() + line.len());
|
||||
out.extend_from_slice(&message[..header.offset_field as usize]);
|
||||
out.extend_from_slice(line.as_bytes());
|
||||
out.extend_from_slice(&message[header.offset_end as usize..]);
|
||||
out
|
||||
}
|
||||
None => {
|
||||
let mut out = line.into_bytes();
|
||||
out.extend_from_slice(message);
|
||||
out
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `prefix` put before the subject, unless it's already there.
|
||||
pub fn prefix_subject(message: &[u8], prefix: &str) -> Option<Vec<u8>> {
|
||||
let parsed = MessageParser::new().parse_headers(message)?;
|
||||
let subject = parsed.subject().unwrap_or_default();
|
||||
if subject.trim_start().starts_with(prefix.trim()) {
|
||||
return None;
|
||||
}
|
||||
Some(set_subject(
|
||||
message,
|
||||
&format!("{} {}", prefix.trim(), subject.trim_start()),
|
||||
))
|
||||
}
|
||||
|
||||
fn escape_html(text: &str) -> String {
|
||||
text.replace('&', "&")
|
||||
.replace('<', "<")
|
||||
.replace('>', ">")
|
||||
.replace('\n', "<br>\n")
|
||||
}
|
||||
|
||||
fn with_text_disclaimer(body: &str, text: &str, position: Position) -> String {
|
||||
let text = text.trim_end();
|
||||
match position {
|
||||
Position::Top => format!("{text}\r\n\r\n{body}"),
|
||||
Position::Bottom => format!("{}\r\n\r\n{text}\r\n", body.trim_end()),
|
||||
}
|
||||
}
|
||||
|
||||
fn with_html_disclaimer(body: &str, html: &str, position: Position) -> String {
|
||||
let lower = body.to_ascii_lowercase();
|
||||
match position {
|
||||
Position::Top => match lower
|
||||
.find("<body")
|
||||
.and_then(|at| lower[at..].find('>').map(|end| at + end + 1))
|
||||
{
|
||||
Some(at) => format!("{}{html}{}", &body[..at], &body[at..]),
|
||||
None => format!("{html}{body}"),
|
||||
},
|
||||
Position::Bottom => match lower.rfind("</body>") {
|
||||
Some(at) => format!("{}{html}{}", &body[..at], &body[at..]),
|
||||
None => format!("{body}{html}"),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// The disclaimer added to each main text and HTML body. `html` is the HTML
|
||||
/// version, or the text escaped when there's none.
|
||||
pub fn add_disclaimer(
|
||||
message: &[u8],
|
||||
text: &str,
|
||||
html: Option<&str>,
|
||||
position: Position,
|
||||
) -> Option<Vec<u8>> {
|
||||
let parsed = MessageParser::new().parse(message)?;
|
||||
let html = html
|
||||
.map(str::to_string)
|
||||
.unwrap_or_else(|| format!("<p>{}</p>", escape_html(text.trim())));
|
||||
let marker = text.trim();
|
||||
|
||||
let mut body_parts: Vec<u32> = parsed
|
||||
.text_body
|
||||
.iter()
|
||||
.chain(parsed.html_body.iter())
|
||||
.copied()
|
||||
.collect();
|
||||
body_parts.sort_unstable();
|
||||
body_parts.dedup();
|
||||
|
||||
// (start, end, replacement) for each part, applied from the last
|
||||
let mut edits: Vec<(usize, usize, Vec<u8>)> = Vec::new();
|
||||
for id in body_parts {
|
||||
let Some(part) = parsed.parts.get(id as usize) else {
|
||||
continue;
|
||||
};
|
||||
let (new_body, content_type) = match &part.body {
|
||||
PartType::Text(body) => {
|
||||
if body.contains(marker) {
|
||||
continue;
|
||||
}
|
||||
(with_text_disclaimer(body, text, position), "text/plain")
|
||||
}
|
||||
PartType::Html(body) => {
|
||||
if body.contains(marker) || body.contains(html.as_str()) {
|
||||
continue;
|
||||
}
|
||||
(with_html_disclaimer(body, &html, position), "text/html")
|
||||
}
|
||||
_ => continue,
|
||||
};
|
||||
// The part's own headers, less the two this changes
|
||||
let mut headers = Vec::new();
|
||||
for header in part.headers() {
|
||||
if matches!(
|
||||
header.name,
|
||||
HeaderName::ContentType | HeaderName::ContentTransferEncoding
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
headers.extend_from_slice(
|
||||
&message[header.offset_field as usize..header.offset_end as usize],
|
||||
);
|
||||
}
|
||||
headers.extend_from_slice(
|
||||
format!("Content-Type: {content_type}; charset=utf-8\r\n").as_bytes(),
|
||||
);
|
||||
headers.extend_from_slice(b"Content-Transfer-Encoding: quoted-printable\r\n\r\n");
|
||||
let encoded = QuotedPrintableEncoder::new()
|
||||
.preserve_line_breaks()
|
||||
.encode(new_body.as_bytes())
|
||||
.ok()?;
|
||||
headers.extend_from_slice(&encoded);
|
||||
// A single-part message's headers are the message's: its first
|
||||
// header is where the part starts
|
||||
let start = part.headers().first().map_or(part.offset_header, |h| {
|
||||
h.offset_field.min(part.offset_header)
|
||||
}) as usize;
|
||||
edits.push((start, part.offset_end as usize, headers));
|
||||
}
|
||||
if edits.is_empty() {
|
||||
return None;
|
||||
}
|
||||
edits.sort_by_key(|(start, _, _)| std::cmp::Reverse(*start));
|
||||
let mut out = message.to_vec();
|
||||
for (start, end, replacement) in edits {
|
||||
out.splice(start..end.min(out.len()), replacement);
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn parse(message: &[u8]) -> mail_parser::Message<'_> {
|
||||
MessageParser::new().parse(message).expect("parses")
|
||||
}
|
||||
|
||||
const PLAIN: &[u8] = b"From: [email protected]\r\nTo: [email protected]\r\nSubject: Hello\r\nContent-Type: text/plain; charset=iso-8859-1\r\nContent-Transfer-Encoding: quoted-printable\r\n\r\nCaf=E9 at noon.\r\n";
|
||||
|
||||
const ALTERNATIVE: &[u8] = b"From: [email protected]\r\nSubject: Plans\r\nMIME-Version: 1.0\r\nContent-Type: multipart/mixed; boundary=\"outer\"\r\n\r\n--outer\r\nContent-Type: multipart/alternative; boundary=\"inner\"\r\n\r\n--inner\r\nContent-Type: text/plain\r\n\r\nSee you.\r\n--inner\r\nContent-Type: text/html\r\nContent-Transfer-Encoding: base64\r\n\r\nPGh0bWw+PGJvZHk+PHA+U2VlIHlvdS48L3A+PC9ib2R5PjwvaHRtbD4=\r\n--inner--\r\n--outer\r\nContent-Type: text/plain; name=\"notes.txt\"\r\nContent-Disposition: attachment; filename=\"notes.txt\"\r\n\r\nAttachment text.\r\n--outer--\r\n";
|
||||
|
||||
#[test]
|
||||
fn headers() {
|
||||
let added = add_header(PLAIN, "X-Mail-Rule", "External");
|
||||
assert_eq!(
|
||||
parse(&added).header_raw("X-Mail-Rule").map(str::trim),
|
||||
Some("External")
|
||||
);
|
||||
let removed = remove_header(&added, "x-mail-rule").unwrap();
|
||||
assert_eq!(removed, PLAIN);
|
||||
assert!(remove_header(PLAIN, "X-Absent").is_none());
|
||||
let utf8 = add_header(PLAIN, "X-Note", "Überprüft");
|
||||
// An RFC 2047 word: mail readers decode it, the wire stays ASCII
|
||||
assert_eq!(
|
||||
parse(&utf8).header_raw("X-Note").map(str::trim),
|
||||
Some("=?utf-8?B?w5xiZXJwcsO8ZnQ=?=")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subjects() {
|
||||
let prefixed = prefix_subject(PLAIN, "[External]").unwrap();
|
||||
assert_eq!(parse(&prefixed).subject(), Some("[External] Hello"));
|
||||
assert!(prefix_subject(&prefixed, "[External]").is_none());
|
||||
let accented = set_subject(PLAIN, "Réunion à midi");
|
||||
assert_eq!(parse(&accented).subject(), Some("Réunion à midi"));
|
||||
assert!(accented.is_ascii(), "encoded as an RFC 2047 word");
|
||||
let none = set_subject(b"From: [email protected]\r\n\r\nBody\r\n", "New");
|
||||
assert_eq!(parse(&none).subject(), Some("New"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disclaimer_on_a_single_part() {
|
||||
let out = add_disclaimer(PLAIN, "Sent by Example Co.", None, Position::Bottom).unwrap();
|
||||
let parsed = parse(&out);
|
||||
let body = parsed.body_text(0).unwrap();
|
||||
assert!(body.starts_with("Café at noon."), "{body:?}");
|
||||
assert!(body.trim_end().ends_with("Sent by Example Co."), "{body:?}");
|
||||
assert_eq!(parsed.subject(), Some("Hello"));
|
||||
assert_eq!(
|
||||
parsed.header_raw("To").map(str::trim),
|
||||
Some("[email protected]")
|
||||
);
|
||||
// Once only
|
||||
assert!(add_disclaimer(&out, "Sent by Example Co.", None, Position::Bottom).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disclaimer_on_alternatives_leaves_attachments() {
|
||||
let out = add_disclaimer(
|
||||
ALTERNATIVE,
|
||||
"Confidential.",
|
||||
Some("<p><i>Confidential.</i></p>"),
|
||||
Position::Top,
|
||||
)
|
||||
.unwrap();
|
||||
let parsed = parse(&out);
|
||||
assert!(
|
||||
parsed
|
||||
.body_text(0)
|
||||
.unwrap()
|
||||
.starts_with("Confidential.\r\n\r\nSee you."),
|
||||
"{:?}",
|
||||
parsed.body_text(0)
|
||||
);
|
||||
let html = parsed.body_html(0).unwrap();
|
||||
assert!(
|
||||
html.contains("<body><p><i>Confidential.</i></p><p>See you.</p>"),
|
||||
"{html}"
|
||||
);
|
||||
assert_eq!(parsed.attachment_count(), 1);
|
||||
assert_eq!(
|
||||
parsed.attachment(0).unwrap().text_contents(),
|
||||
Some("Attachment text.")
|
||||
);
|
||||
assert!(!String::from_utf8_lossy(&out).contains("Confidential.\r\n\r\nAttachment"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,770 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! Mail flow rules and DLP rules (dlp-and-mail-flow-rules spec, §2.2–§2.4):
|
||||
//! what a rule is, what makes one valid, and where it's kept.
|
||||
//!
|
||||
//! Kept in the fork's subspace (`store::SUBSPACE_INBUXA`), never in the
|
||||
//! registry, so an upstream schema import never touches them. Every key
|
||||
//! starts with `R`, then one byte for the kind:
|
||||
//!
|
||||
//! - `r` + rule id (u32): the rule, as JSON.
|
||||
//!
|
||||
//! Numbers are big-endian. There are few rules, so they're read whole.
|
||||
|
||||
use super::{detectors, words};
|
||||
use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize, de::DeserializeOwned};
|
||||
use store::{
|
||||
Deserialize, IterateParams, SUBSPACE_INBUXA, Serialize, Store, ValueKey,
|
||||
write::{AnyClass, BatchBuilder, ValueClass, assert::AssertValue},
|
||||
};
|
||||
use trc::AddContext;
|
||||
|
||||
const FEATURE: u8 = b'R';
|
||||
const KIND_RULE: u8 = b'r';
|
||||
const CREATE_ATTEMPTS: usize = 5;
|
||||
|
||||
/// Longest text a rule may carry (a notice, a disclaimer), in bytes.
|
||||
const MAX_TEXT: usize = 16 * 1024;
|
||||
/// Most entries in one list (words, addresses, domains).
|
||||
const MAX_LIST: usize = 5_000;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum Kind {
|
||||
Dlp,
|
||||
Transport,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum Direction {
|
||||
/// Mail an authenticated sender submits, over SMTP or JMAP.
|
||||
Outgoing,
|
||||
/// Everything else the server accepts.
|
||||
Incoming,
|
||||
Any,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum Position {
|
||||
Top,
|
||||
Bottom,
|
||||
}
|
||||
|
||||
fn one() -> u32 {
|
||||
1
|
||||
}
|
||||
|
||||
/// Group and tenant ids in the JMAP form clients use (`"b"`, `"c"`…), held
|
||||
/// as numbers for matching. Plain numbers are read too.
|
||||
pub(crate) mod jmap_ids {
|
||||
use serde::{Deserialize, Deserializer, Serializer, de::Error, ser::SerializeSeq};
|
||||
use std::str::FromStr;
|
||||
use types::id::Id;
|
||||
|
||||
pub fn serialize<S: Serializer>(ids: &[u32], serializer: S) -> Result<S::Ok, S::Error> {
|
||||
let mut seq = serializer.serialize_seq(Some(ids.len()))?;
|
||||
for id in ids {
|
||||
seq.serialize_element(&Id::from(*id).to_string())?;
|
||||
}
|
||||
seq.end()
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum Either {
|
||||
Text(String),
|
||||
Number(u32),
|
||||
}
|
||||
|
||||
pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Vec<u32>, D::Error> {
|
||||
Vec::<Either>::deserialize(deserializer)?
|
||||
.into_iter()
|
||||
.map(|id| match id {
|
||||
Either::Number(n) => Ok(n),
|
||||
Either::Text(text) => Id::from_str(&text)
|
||||
.map(|id| id.document_id())
|
||||
.map_err(|_| D::Error::custom(format!("\"{text}\" isn't an id"))),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// A detector and the least it must find.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DetectorMin {
|
||||
pub id: String,
|
||||
#[serde(default = "one")]
|
||||
pub at_least: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(
|
||||
tag = "type",
|
||||
rename_all = "camelCase",
|
||||
rename_all_fields = "camelCase"
|
||||
)]
|
||||
pub enum Condition {
|
||||
SenderAddress {
|
||||
addresses: Vec<String>,
|
||||
},
|
||||
SenderDomain {
|
||||
domains: Vec<String>,
|
||||
},
|
||||
SenderGroup {
|
||||
#[serde(with = "jmap_ids")]
|
||||
groups: Vec<u32>,
|
||||
},
|
||||
SenderTenant {
|
||||
#[serde(with = "jmap_ids")]
|
||||
tenants: Vec<u32>,
|
||||
},
|
||||
/// Any recipient is one of these.
|
||||
RecipientAddress {
|
||||
addresses: Vec<String>,
|
||||
},
|
||||
RecipientDomain {
|
||||
domains: Vec<String>,
|
||||
},
|
||||
RecipientGroup {
|
||||
#[serde(with = "jmap_ids")]
|
||||
groups: Vec<u32>,
|
||||
},
|
||||
/// Any recipient isn't at a domain this server hosts.
|
||||
RecipientOutside,
|
||||
/// Words or phrases in the subject, body or readable attachments.
|
||||
Words {
|
||||
words: Vec<String>,
|
||||
#[serde(default = "one")]
|
||||
at_least: u32,
|
||||
},
|
||||
/// The organization's regular expression, in the same places.
|
||||
Pattern {
|
||||
pattern: String,
|
||||
#[serde(default = "one")]
|
||||
at_least: u32,
|
||||
},
|
||||
/// A header exists, or its value contains or matches.
|
||||
Header {
|
||||
name: String,
|
||||
#[serde(default)]
|
||||
contains: Option<String>,
|
||||
#[serde(default)]
|
||||
matches: Option<String>,
|
||||
},
|
||||
/// An attachment's declared or detected type starts with one of these.
|
||||
AttachmentType {
|
||||
types: Vec<String>,
|
||||
},
|
||||
AttachmentExtension {
|
||||
extensions: Vec<String>,
|
||||
},
|
||||
AttachmentName {
|
||||
pattern: String,
|
||||
},
|
||||
AttachmentSizeOver {
|
||||
bytes: u64,
|
||||
},
|
||||
AttachmentCountOver {
|
||||
count: u32,
|
||||
},
|
||||
/// An attachment is encrypted, a PDF, a legacy Office file, an archive
|
||||
/// inside an archive, or past the inspection limit.
|
||||
CantBeInspected,
|
||||
MessageSizeOver {
|
||||
bytes: u64,
|
||||
},
|
||||
/// Any of these detectors finds at least its minimum (DLP rules only).
|
||||
Detected {
|
||||
detectors: Vec<DetectorMin>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(
|
||||
tag = "type",
|
||||
rename_all = "camelCase",
|
||||
rename_all_fields = "camelCase"
|
||||
)]
|
||||
pub enum Action {
|
||||
// Transport actions
|
||||
AddDisclaimer {
|
||||
text: String,
|
||||
#[serde(default)]
|
||||
html: Option<String>,
|
||||
position: Position,
|
||||
},
|
||||
AddHeader {
|
||||
name: String,
|
||||
value: String,
|
||||
},
|
||||
RemoveHeader {
|
||||
name: String,
|
||||
},
|
||||
PrefixSubject {
|
||||
text: String,
|
||||
},
|
||||
AddRecipient {
|
||||
address: String,
|
||||
},
|
||||
Redirect {
|
||||
addresses: Vec<String>,
|
||||
},
|
||||
Refuse {
|
||||
text: String,
|
||||
},
|
||||
Route {
|
||||
queue: String,
|
||||
},
|
||||
// DLP actions
|
||||
Block {
|
||||
notice: String,
|
||||
},
|
||||
Warn {
|
||||
notice: String,
|
||||
},
|
||||
Hold {
|
||||
notice: String,
|
||||
#[serde(default)]
|
||||
notify_sender: bool,
|
||||
},
|
||||
}
|
||||
|
||||
impl Action {
|
||||
pub fn is_dlp(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Action::Block { .. } | Action::Warn { .. } | Action::Hold { .. }
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Rule {
|
||||
#[serde(default)]
|
||||
pub id: u32,
|
||||
pub name: String,
|
||||
#[serde(default)]
|
||||
pub description: String,
|
||||
pub kind: Kind,
|
||||
#[serde(default = "enabled")]
|
||||
pub enabled: bool,
|
||||
#[serde(default)]
|
||||
pub priority: i32,
|
||||
pub direction: Direction,
|
||||
#[serde(default)]
|
||||
pub conditions: Vec<Condition>,
|
||||
#[serde(default)]
|
||||
pub exceptions: Vec<Condition>,
|
||||
pub actions: Vec<Action>,
|
||||
#[serde(default)]
|
||||
pub stop_processing: bool,
|
||||
#[serde(default)]
|
||||
pub created_by: String,
|
||||
#[serde(default)]
|
||||
pub created_at: u64,
|
||||
#[serde(default)]
|
||||
pub updated_at: u64,
|
||||
}
|
||||
|
||||
fn enabled() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
/// Why a rule can't be saved: the property at fault, and a sentence.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Invalid {
|
||||
pub property: &'static str,
|
||||
pub reason: String,
|
||||
}
|
||||
|
||||
fn invalid(property: &'static str, reason: impl Into<String>) -> Invalid {
|
||||
Invalid {
|
||||
property,
|
||||
reason: reason.into(),
|
||||
}
|
||||
}
|
||||
|
||||
impl Rule {
|
||||
/// Everything that can be checked without the rest of the server: the
|
||||
/// shape (§2.2, §2.4), the detectors, word lists and patterns.
|
||||
pub fn validate(&self) -> Result<(), Invalid> {
|
||||
if self.name.trim().is_empty() {
|
||||
return Err(invalid("name", "A rule needs a name."));
|
||||
}
|
||||
if self.name.len() > 200 || self.description.len() > MAX_TEXT {
|
||||
return Err(invalid("name", "The name or description is too long."));
|
||||
}
|
||||
if self.actions.is_empty() {
|
||||
return Err(invalid("actions", "A rule needs something to do."));
|
||||
}
|
||||
let dlp_actions = self.actions.iter().filter(|a| a.is_dlp()).count();
|
||||
match self.kind {
|
||||
Kind::Dlp => {
|
||||
if self.direction != Direction::Outgoing {
|
||||
return Err(invalid("direction", "DLP rules check outgoing mail only."));
|
||||
}
|
||||
if dlp_actions != 1 || self.actions.len() != 1 {
|
||||
return Err(invalid(
|
||||
"actions",
|
||||
"A DLP rule has exactly one action: block, warn or hold.",
|
||||
));
|
||||
}
|
||||
}
|
||||
Kind::Transport => {
|
||||
if dlp_actions > 0 {
|
||||
return Err(invalid(
|
||||
"actions",
|
||||
"Block, warn and hold belong to DLP rules.",
|
||||
));
|
||||
}
|
||||
if self
|
||||
.conditions
|
||||
.iter()
|
||||
.chain(&self.exceptions)
|
||||
.any(|c| matches!(c, Condition::Detected { .. }))
|
||||
{
|
||||
return Err(invalid("conditions", "Detectors belong to DLP rules."));
|
||||
}
|
||||
}
|
||||
}
|
||||
for (property, list) in [
|
||||
("conditions", &self.conditions),
|
||||
("exceptions", &self.exceptions),
|
||||
] {
|
||||
for condition in list {
|
||||
validate_condition(condition).map_err(|reason| invalid(property, reason))?;
|
||||
}
|
||||
}
|
||||
for action in &self.actions {
|
||||
validate_action(action).map_err(|reason| invalid("actions", reason))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn nonempty_list<T>(list: &[T], what: &str) -> Result<(), String> {
|
||||
if list.is_empty() {
|
||||
Err(format!("The {what} list is empty."))
|
||||
} else if list.len() > MAX_LIST {
|
||||
Err(format!(
|
||||
"The {what} list is longer than {MAX_LIST} entries."
|
||||
))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn header_name(name: &str) -> Result<(), String> {
|
||||
if !name.is_empty()
|
||||
&& name.len() <= 100
|
||||
&& name.bytes().all(|b| b.is_ascii_graphic() && b != b':')
|
||||
{
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!("\"{name}\" isn't a header name."))
|
||||
}
|
||||
}
|
||||
|
||||
fn text(value: &str, what: &str) -> Result<(), String> {
|
||||
if value.trim().is_empty() {
|
||||
Err(format!("The {what} is empty."))
|
||||
} else if value.len() > MAX_TEXT {
|
||||
Err(format!("The {what} is longer than {MAX_TEXT} bytes."))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_condition(condition: &Condition) -> Result<(), String> {
|
||||
match condition {
|
||||
Condition::SenderAddress { addresses } | Condition::RecipientAddress { addresses } => {
|
||||
nonempty_list(addresses, "address")
|
||||
}
|
||||
Condition::SenderDomain { domains } | Condition::RecipientDomain { domains } => {
|
||||
nonempty_list(domains, "domain")
|
||||
}
|
||||
Condition::SenderGroup { groups } | Condition::RecipientGroup { groups } => {
|
||||
nonempty_list(groups, "group")
|
||||
}
|
||||
Condition::SenderTenant { tenants } => nonempty_list(tenants, "tenant"),
|
||||
Condition::Words { words, at_least } => {
|
||||
nonempty_list(words, "word")?;
|
||||
if *at_least == 0 {
|
||||
return Err("The least number of words must be 1 or more.".into());
|
||||
}
|
||||
words::WordList::new(words).map(|_| ())
|
||||
}
|
||||
Condition::Pattern { pattern, at_least } => {
|
||||
if *at_least == 0 {
|
||||
return Err("The least number of matches must be 1 or more.".into());
|
||||
}
|
||||
words::Pattern::new(pattern).map(|_| ())
|
||||
}
|
||||
Condition::Header {
|
||||
name,
|
||||
contains,
|
||||
matches,
|
||||
} => {
|
||||
header_name(name)?;
|
||||
if let Some(pattern) = matches {
|
||||
words::Pattern::new(pattern)?;
|
||||
}
|
||||
if contains.is_some() && matches.is_some() {
|
||||
return Err("A header condition is either contains or matches.".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Condition::AttachmentType { types } => nonempty_list(types, "type"),
|
||||
Condition::AttachmentExtension { extensions } => nonempty_list(extensions, "extension"),
|
||||
Condition::AttachmentName { pattern } => words::Pattern::new(pattern).map(|_| ()),
|
||||
Condition::Detected { detectors } => {
|
||||
nonempty_list(detectors, "detector")?;
|
||||
for d in detectors {
|
||||
if detectors::by_id(&d.id).is_none() {
|
||||
return Err(format!("There is no detector \"{}\".", d.id));
|
||||
}
|
||||
if d.at_least == 0 {
|
||||
return Err("A detector's least count must be 1 or more.".into());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Condition::RecipientOutside
|
||||
| Condition::AttachmentSizeOver { .. }
|
||||
| Condition::AttachmentCountOver { .. }
|
||||
| Condition::CantBeInspected
|
||||
| Condition::MessageSizeOver { .. } => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_action(action: &Action) -> Result<(), String> {
|
||||
match action {
|
||||
Action::AddDisclaimer { text: t, html, .. } => {
|
||||
text(t, "disclaimer")?;
|
||||
html.as_deref()
|
||||
.map_or(Ok(()), |h| text(h, "disclaimer's HTML"))
|
||||
}
|
||||
Action::AddHeader { name, value } => {
|
||||
header_name(name)?;
|
||||
if value.len() > 998 || value.contains(['\r', '\n']) {
|
||||
Err("A header value is one line of at most 998 characters.".into())
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
Action::RemoveHeader { name } => header_name(name),
|
||||
Action::PrefixSubject { text: t } => text(t, "subject prefix"),
|
||||
Action::AddRecipient { address } => {
|
||||
if address.contains('@') {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!("\"{address}\" isn't an address."))
|
||||
}
|
||||
}
|
||||
Action::Redirect { addresses } => {
|
||||
nonempty_list(addresses, "address")?;
|
||||
match addresses.iter().find(|a| !a.contains('@')) {
|
||||
Some(a) => Err(format!("\"{a}\" isn't an address.")),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
Action::Refuse { text: t } => text(t, "refusal text"),
|
||||
Action::Route { queue } => text(queue, "queue"),
|
||||
Action::Block { notice } | Action::Warn { notice } | Action::Hold { notice, .. } => {
|
||||
text(notice, "notice")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Storage --------------------------------------------------------------
|
||||
|
||||
struct Json<T>(T);
|
||||
|
||||
impl<T: SerdeSerialize> Serialize for Json<T> {
|
||||
fn serialize(&self) -> trc::Result<Vec<u8>> {
|
||||
serde_json::to_vec(&self.0).map_err(|err| {
|
||||
trc::StoreEvent::UnexpectedError
|
||||
.into_err()
|
||||
.details("Failed to serialize mail rule")
|
||||
.reason(err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: DeserializeOwned + Sync + Send> Deserialize for Json<T> {
|
||||
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
|
||||
serde_json::from_slice(bytes).map(Json).map_err(|err| {
|
||||
trc::StoreEvent::DataCorruption
|
||||
.into_err()
|
||||
.details("Invalid mail rule")
|
||||
.reason(err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn class(id: u32) -> ValueClass {
|
||||
let mut key = Vec::with_capacity(6);
|
||||
key.push(FEATURE);
|
||||
key.push(KIND_RULE);
|
||||
key.extend_from_slice(&id.to_be_bytes());
|
||||
ValueClass::Any(AnyClass {
|
||||
subspace: SUBSPACE_INBUXA,
|
||||
key,
|
||||
})
|
||||
}
|
||||
|
||||
fn key(id: u32) -> ValueKey<ValueClass> {
|
||||
ValueKey::from(class(id))
|
||||
}
|
||||
|
||||
pub async fn get(data: &Store, id: u32) -> trc::Result<Option<Rule>> {
|
||||
Ok(data
|
||||
.get_value::<Json<Rule>>(key(id))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.map(|Json(rule)| rule))
|
||||
}
|
||||
|
||||
/// Every rule, in the order they run: by priority, then oldest first.
|
||||
pub async fn all(data: &Store) -> trc::Result<Vec<Rule>> {
|
||||
let mut rules = Vec::new();
|
||||
data.iterate(IterateParams::new(key(0), key(u32::MAX)), |_, value| {
|
||||
if let Ok(Json(rule)) = Json::<Rule>::deserialize(value) {
|
||||
rules.push(rule);
|
||||
}
|
||||
Ok(true)
|
||||
})
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
rules.sort_by_key(|rule| (rule.priority, rule.id));
|
||||
Ok(rules)
|
||||
}
|
||||
|
||||
/// Writes a new rule under the next free id, which it returns. Two nodes
|
||||
/// creating rules at once can't take the same id: the key must be absent.
|
||||
pub async fn create(data: &Store, rule: &Rule) -> trc::Result<u32> {
|
||||
let mut attempt = 0;
|
||||
loop {
|
||||
attempt += 1;
|
||||
let id = all(data).await?.iter().map(|r| r.id).max().unwrap_or(0) + 1;
|
||||
let stored = Rule { id, ..rule.clone() };
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.assert_value(class(id), AssertValue::None);
|
||||
batch.set(class(id), Json(&stored).serialize()?);
|
||||
match data.write(batch.build_all()).await {
|
||||
Ok(_) => {
|
||||
super::cache::invalidate();
|
||||
return Ok(id);
|
||||
}
|
||||
Err(err)
|
||||
if attempt < CREATE_ATTEMPTS
|
||||
&& matches!(
|
||||
err.as_ref(),
|
||||
trc::EventType::Store(trc::StoreEvent::AssertValueFailed)
|
||||
) => {}
|
||||
Err(err) => return Err(err.caused_by(trc::location!())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Replaces a stored rule (same id).
|
||||
pub async fn update(data: &Store, rule: &Rule) -> trc::Result<()> {
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.set(class(rule.id), Json(rule).serialize()?);
|
||||
data.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
super::cache::invalidate();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn delete(data: &Store, id: u32) -> trc::Result<()> {
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.clear(class(id));
|
||||
data.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
super::cache::invalidate();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn rule(kind: Kind, actions: Vec<Action>) -> Rule {
|
||||
Rule {
|
||||
id: 0,
|
||||
name: "Cards outside".into(),
|
||||
description: String::new(),
|
||||
kind,
|
||||
enabled: true,
|
||||
priority: 0,
|
||||
direction: Direction::Outgoing,
|
||||
conditions: vec![Condition::RecipientOutside],
|
||||
exceptions: vec![],
|
||||
actions,
|
||||
stop_processing: false,
|
||||
created_by: String::new(),
|
||||
created_at: 0,
|
||||
updated_at: 0,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wire_format() {
|
||||
let json = r#"{"name":"Cards","kind":"dlp","direction":"outgoing",
|
||||
"conditions":[{"type":"recipientOutside"},{"type":"detected","detectors":[{"id":"payment-card","atLeast":5}]}],
|
||||
"actions":[{"type":"hold","notice":"Held for review","notifySender":true}]}"#;
|
||||
let parsed: Rule = serde_json::from_str(json).unwrap();
|
||||
assert!(parsed.enabled);
|
||||
assert_eq!(
|
||||
parsed.conditions[1],
|
||||
Condition::Detected {
|
||||
detectors: vec![DetectorMin {
|
||||
id: "payment-card".into(),
|
||||
at_least: 5
|
||||
}]
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
parsed.actions[0],
|
||||
Action::Hold {
|
||||
notice: "Held for review".into(),
|
||||
notify_sender: true
|
||||
}
|
||||
);
|
||||
assert!(parsed.validate().is_ok());
|
||||
let back = serde_json::to_value(&parsed).unwrap();
|
||||
assert_eq!(back["actions"][0]["notifySender"], true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn group_and_tenant_ids_are_jmap_ids() {
|
||||
let condition: Condition =
|
||||
serde_json::from_str(r#"{"type":"senderGroup","groups":["b", 7]}"#).unwrap();
|
||||
assert_eq!(condition, Condition::SenderGroup { groups: vec![1, 7] });
|
||||
assert_eq!(
|
||||
serde_json::to_value(&condition).unwrap()["groups"],
|
||||
serde_json::json!(["b", "h"])
|
||||
);
|
||||
assert!(
|
||||
serde_json::from_str::<Condition>(r#"{"type":"senderTenant","tenants":["!!"]}"#)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dlp_rules_have_one_dlp_action_on_outgoing_mail() {
|
||||
let block = Action::Block {
|
||||
notice: "No.".into(),
|
||||
};
|
||||
assert!(rule(Kind::Dlp, vec![block.clone()]).validate().is_ok());
|
||||
let two = rule(
|
||||
Kind::Dlp,
|
||||
vec![
|
||||
block.clone(),
|
||||
Action::Warn {
|
||||
notice: "Hm.".into(),
|
||||
},
|
||||
],
|
||||
);
|
||||
assert_eq!(two.validate().unwrap_err().property, "actions");
|
||||
let mixed = rule(
|
||||
Kind::Dlp,
|
||||
vec![block.clone(), Action::PrefixSubject { text: "[x]".into() }],
|
||||
);
|
||||
assert_eq!(mixed.validate().unwrap_err().property, "actions");
|
||||
let mut inbound = rule(Kind::Dlp, vec![block.clone()]);
|
||||
inbound.direction = Direction::Incoming;
|
||||
assert_eq!(inbound.validate().unwrap_err().property, "direction");
|
||||
assert_eq!(
|
||||
rule(Kind::Transport, vec![block])
|
||||
.validate()
|
||||
.unwrap_err()
|
||||
.property,
|
||||
"actions"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn conditions_and_actions_are_checked() {
|
||||
let disclaimer = Action::AddDisclaimer {
|
||||
text: "Sent from Example Co.".into(),
|
||||
html: None,
|
||||
position: Position::Bottom,
|
||||
};
|
||||
let mut r = rule(Kind::Transport, vec![disclaimer]);
|
||||
assert!(r.validate().is_ok());
|
||||
r.conditions.push(Condition::Detected {
|
||||
detectors: vec![DetectorMin {
|
||||
id: "iban".into(),
|
||||
at_least: 1,
|
||||
}],
|
||||
});
|
||||
assert_eq!(r.validate().unwrap_err().property, "conditions");
|
||||
|
||||
let mut r = rule(
|
||||
Kind::Dlp,
|
||||
vec![Action::Block {
|
||||
notice: "No.".into(),
|
||||
}],
|
||||
);
|
||||
r.conditions = vec![Condition::Detected {
|
||||
detectors: vec![DetectorMin {
|
||||
id: "nope".into(),
|
||||
at_least: 1,
|
||||
}],
|
||||
}];
|
||||
assert!(r.validate().unwrap_err().reason.contains("nope"));
|
||||
r.conditions = vec![Condition::Pattern {
|
||||
pattern: "(".into(),
|
||||
at_least: 1,
|
||||
}];
|
||||
assert!(r.validate().is_err());
|
||||
r.conditions = vec![Condition::Words {
|
||||
words: vec![],
|
||||
at_least: 1,
|
||||
}];
|
||||
assert!(r.validate().is_err());
|
||||
r.exceptions = vec![Condition::Header {
|
||||
name: "X-Bad: yes".into(),
|
||||
contains: None,
|
||||
matches: None,
|
||||
}];
|
||||
r.conditions = vec![];
|
||||
assert_eq!(r.validate().unwrap_err().property, "exceptions");
|
||||
|
||||
let header = rule(
|
||||
Kind::Transport,
|
||||
vec![Action::AddHeader {
|
||||
name: "X-Tag".into(),
|
||||
value: "a\r\nBcc: x@y".into(),
|
||||
}],
|
||||
);
|
||||
assert!(header.validate().is_err());
|
||||
let redirect = rule(
|
||||
Kind::Transport,
|
||||
vec![Action::Redirect {
|
||||
addresses: vec!["nobody".into()],
|
||||
}],
|
||||
);
|
||||
assert!(redirect.validate().is_err());
|
||||
let mut unnamed = rule(
|
||||
Kind::Transport,
|
||||
vec![Action::RemoveHeader {
|
||||
name: "X-Tag".into(),
|
||||
}],
|
||||
);
|
||||
unnamed.name = " ".into();
|
||||
assert_eq!(unnamed.validate().unwrap_err().property, "name");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! An organization's own word lists and patterns (§2.3). Both count
|
||||
//! occurrences, not distinct values: "confidential" three times is three.
|
||||
|
||||
use aho_corasick::{AhoCorasick, AhoCorasickBuilder, MatchKind};
|
||||
use regex::{Regex, RegexBuilder};
|
||||
|
||||
/// How large a compiled pattern may grow. Keeps a rule someone writes from
|
||||
/// making every message slow to send.
|
||||
const PATTERN_SIZE_LIMIT: usize = 1 << 20;
|
||||
|
||||
/// Words and phrases, matched whole and ignoring case.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct WordList {
|
||||
matcher: AhoCorasick,
|
||||
}
|
||||
|
||||
impl WordList {
|
||||
/// Builds a list from words or phrases; empty entries are skipped.
|
||||
pub fn new<I, S>(words: I) -> Result<Self, String>
|
||||
where
|
||||
I: IntoIterator<Item = S>,
|
||||
S: AsRef<str>,
|
||||
{
|
||||
let words: Vec<String> = words
|
||||
.into_iter()
|
||||
.map(|w| w.as_ref().trim().to_lowercase())
|
||||
.filter(|w| !w.is_empty())
|
||||
.collect();
|
||||
if words.is_empty() {
|
||||
return Err("The list has no words".into());
|
||||
}
|
||||
AhoCorasickBuilder::new()
|
||||
.match_kind(MatchKind::LeftmostLongest)
|
||||
.build(&words)
|
||||
.map(|matcher| Self { matcher })
|
||||
.map_err(|err| err.to_string())
|
||||
}
|
||||
|
||||
/// How many times any word of the list appears in `text`.
|
||||
pub fn count(&self, text: &str) -> usize {
|
||||
let text = text.to_lowercase();
|
||||
self.matcher
|
||||
.find_iter(&text)
|
||||
.filter(|m| super::detectors::stands_alone(&text, m.start(), m.end()))
|
||||
.count()
|
||||
}
|
||||
}
|
||||
|
||||
/// An organization's regular expression.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Pattern {
|
||||
regex: Regex,
|
||||
}
|
||||
|
||||
impl Pattern {
|
||||
/// Compiles `pattern`, or says why it can't be used. Matching ignores
|
||||
/// case unless the pattern turns that off with `(?-i)`.
|
||||
pub fn new(pattern: &str) -> Result<Self, String> {
|
||||
RegexBuilder::new(pattern)
|
||||
.case_insensitive(true)
|
||||
.size_limit(PATTERN_SIZE_LIMIT)
|
||||
.build()
|
||||
.map(|regex| Self { regex })
|
||||
.map_err(|err| err.to_string())
|
||||
}
|
||||
|
||||
/// How many times the pattern matches in `text`.
|
||||
pub fn count(&self, text: &str) -> usize {
|
||||
self.regex.find_iter(text).filter(|m| !m.is_empty()).count()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn words_whole_and_any_case() {
|
||||
let list = WordList::new(["Project Falcon", "confidential", " "]).unwrap();
|
||||
assert_eq!(
|
||||
list.count(
|
||||
"CONFIDENTIAL: project falcon notes. Not confidentiality, not projectfalcon."
|
||||
),
|
||||
2
|
||||
);
|
||||
assert_eq!(list.count("Confidential, confidential and confidential"), 3);
|
||||
// Non-ASCII case folding
|
||||
let list = WordList::new(["GEHEIM", "Straße"]).unwrap();
|
||||
assert_eq!(list.count("streng geheim, STRASSE ist nicht Straße"), 2);
|
||||
assert!(WordList::new(["", " "]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn patterns() {
|
||||
let pattern = Pattern::new(r"\bPRJ-\d{4}\b").unwrap();
|
||||
assert_eq!(pattern.count("prj-1234 and PRJ-5678, not PRJ-12"), 2);
|
||||
assert!(Pattern::new("(unclosed").is_err());
|
||||
// Too large to compile within the limit
|
||||
assert!(Pattern::new(r"\w{1000}\w{1000}\w{1000}").is_err());
|
||||
// Empty matches don't count
|
||||
assert_eq!(Pattern::new("x*").unwrap().count("abc"), 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! Accepted security to-do items (security to-do list spec, SS-23 to SS-26).
|
||||
//!
|
||||
//! The console runs the checks; the server only keeps what an administrator
|
||||
//! accepted, so every administrator sees the same accepted risks. An
|
||||
//! acceptance names the check, what within it (a domain, a certificate…),
|
||||
//! the value the check saw, and why. It holds only while the check still
|
||||
//! sees that value, which the console compares. Acceptances are created and
|
||||
//! removed, never edited.
|
||||
//!
|
||||
//! Kept in the fork's subspace (`store::SUBSPACE_INBUXA`). Every key starts
|
||||
//! with `Q`, then one byte for the kind:
|
||||
//!
|
||||
//! - `a` + acceptance id (u32): the acceptance, as JSON.
|
||||
//!
|
||||
//! Numbers are big-endian. There are at most [`MAX_ACCEPTANCES`], so
|
||||
//! they're read whole.
|
||||
|
||||
use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize};
|
||||
use store::{
|
||||
Deserialize, IterateParams, SUBSPACE_INBUXA, Serialize, Store, ValueKey,
|
||||
write::{AnyClass, BatchBuilder, ValueClass, assert::AssertValue},
|
||||
};
|
||||
use trc::AddContext;
|
||||
|
||||
const FEATURE: u8 = b'Q';
|
||||
const KIND_ACCEPTANCE: u8 = b'a';
|
||||
const CREATE_ATTEMPTS: usize = 5;
|
||||
|
||||
pub const MAX_ACCEPTANCES: usize = 200;
|
||||
/// The checks are SS-1 to SS-18; a few spare for checks added later.
|
||||
const MAX_CHECK: u32 = 40;
|
||||
const MAX_SUBJECT: usize = 255;
|
||||
const MAX_VALUE_BYTES: usize = 4096;
|
||||
const MAX_NOTE: usize = 500;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Acceptance {
|
||||
#[serde(default)]
|
||||
pub id: u32,
|
||||
/// Which check: `SS-1`, `SS-2`…
|
||||
pub check: String,
|
||||
/// What within the check: empty for a server-wide setting, else the
|
||||
/// domain, strategy or certificate it names.
|
||||
#[serde(default)]
|
||||
pub subject: String,
|
||||
/// The value the check saw when it was accepted.
|
||||
#[serde(default)]
|
||||
pub accepted_value: serde_json::Value,
|
||||
/// Why. Required.
|
||||
pub note: String,
|
||||
#[serde(default)]
|
||||
pub accepted_by: String,
|
||||
/// Seconds since the epoch.
|
||||
#[serde(default)]
|
||||
pub accepted_at: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub struct Invalid {
|
||||
pub property: &'static str,
|
||||
pub reason: String,
|
||||
}
|
||||
|
||||
fn invalid(property: &'static str, reason: impl Into<String>) -> Invalid {
|
||||
Invalid {
|
||||
property,
|
||||
reason: reason.into(),
|
||||
}
|
||||
}
|
||||
|
||||
impl Acceptance {
|
||||
/// What an administrator sends is checked whole before it's kept.
|
||||
pub fn validate(&self) -> Result<(), Invalid> {
|
||||
let check_ok = self
|
||||
.check
|
||||
.strip_prefix("SS-")
|
||||
.and_then(|n| n.parse::<u32>().ok())
|
||||
.is_some_and(|n| (1..=MAX_CHECK).contains(&n));
|
||||
if !check_ok {
|
||||
return Err(invalid("check", "A check is named SS-1, SS-2 and so on."));
|
||||
}
|
||||
if self.subject.chars().count() > MAX_SUBJECT {
|
||||
return Err(invalid(
|
||||
"subject",
|
||||
format!("At most {MAX_SUBJECT} characters."),
|
||||
));
|
||||
}
|
||||
let value_bytes = serde_json::to_vec(&self.accepted_value)
|
||||
.map(|v| v.len())
|
||||
.unwrap_or(usize::MAX);
|
||||
if value_bytes > MAX_VALUE_BYTES {
|
||||
return Err(invalid(
|
||||
"acceptedValue",
|
||||
format!("At most {MAX_VALUE_BYTES} bytes."),
|
||||
));
|
||||
}
|
||||
let note = self.note.trim();
|
||||
if note.is_empty() {
|
||||
return Err(invalid("note", "Say why this is accepted."));
|
||||
}
|
||||
if note.chars().count() > MAX_NOTE {
|
||||
return Err(invalid("note", format!("At most {MAX_NOTE} characters.")));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// --- Storage --------------------------------------------------------------
|
||||
|
||||
struct Json<T>(T);
|
||||
|
||||
impl<T: SerdeSerialize> Serialize for Json<T> {
|
||||
fn serialize(&self) -> trc::Result<Vec<u8>> {
|
||||
serde_json::to_vec(&self.0).map_err(|err| {
|
||||
trc::StoreEvent::UnexpectedError
|
||||
.into_err()
|
||||
.details("Failed to serialize a security acceptance")
|
||||
.reason(err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Deserialize for Json<Acceptance> {
|
||||
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
|
||||
serde_json::from_slice(bytes).map(Json).map_err(|err| {
|
||||
trc::StoreEvent::DataCorruption
|
||||
.into_err()
|
||||
.details("Invalid security acceptance")
|
||||
.reason(err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn class(id: u32) -> ValueClass {
|
||||
let mut key = Vec::with_capacity(6);
|
||||
key.push(FEATURE);
|
||||
key.push(KIND_ACCEPTANCE);
|
||||
key.extend_from_slice(&id.to_be_bytes());
|
||||
ValueClass::Any(AnyClass {
|
||||
subspace: SUBSPACE_INBUXA,
|
||||
key,
|
||||
})
|
||||
}
|
||||
|
||||
fn key(id: u32) -> ValueKey<ValueClass> {
|
||||
ValueKey::from(class(id))
|
||||
}
|
||||
|
||||
pub async fn get(data: &Store, id: u32) -> trc::Result<Option<Acceptance>> {
|
||||
Ok(data
|
||||
.get_value::<Json<Acceptance>>(key(id))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.map(|Json(acceptance)| acceptance))
|
||||
}
|
||||
|
||||
/// Every acceptance, oldest first.
|
||||
pub async fn all(data: &Store) -> trc::Result<Vec<Acceptance>> {
|
||||
let mut out = Vec::new();
|
||||
data.iterate(IterateParams::new(key(0), key(u32::MAX)), |_, value| {
|
||||
if let Ok(Json(acceptance)) = Json::<Acceptance>::deserialize(value) {
|
||||
out.push(acceptance);
|
||||
}
|
||||
Ok(true)
|
||||
})
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
out.sort_by_key(|a| a.id);
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
pub enum Created {
|
||||
Id(u32),
|
||||
/// There are already [`MAX_ACCEPTANCES`].
|
||||
Full,
|
||||
}
|
||||
|
||||
/// Keeps a new acceptance under the next free id. Two nodes creating at
|
||||
/// once can't take the same id: the key must be absent.
|
||||
pub async fn create(data: &Store, acceptance: &Acceptance) -> trc::Result<Created> {
|
||||
let mut attempt = 0;
|
||||
loop {
|
||||
attempt += 1;
|
||||
let existing = all(data).await?;
|
||||
if existing.len() >= MAX_ACCEPTANCES {
|
||||
return Ok(Created::Full);
|
||||
}
|
||||
let id = existing.iter().map(|a| a.id).max().unwrap_or(0) + 1;
|
||||
let stored = Acceptance {
|
||||
id,
|
||||
..acceptance.clone()
|
||||
};
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.assert_value(class(id), AssertValue::None);
|
||||
batch.set(class(id), Json(&stored).serialize()?);
|
||||
match data.write(batch.build_all()).await {
|
||||
Ok(_) => return Ok(Created::Id(id)),
|
||||
Err(err)
|
||||
if attempt < CREATE_ATTEMPTS
|
||||
&& matches!(
|
||||
err.as_ref(),
|
||||
trc::EventType::Store(trc::StoreEvent::AssertValueFailed)
|
||||
) => {}
|
||||
Err(err) => return Err(err.caused_by(trc::location!())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn delete(data: &Store, id: u32) -> trc::Result<()> {
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.clear(class(id));
|
||||
data.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn acceptance() -> Acceptance {
|
||||
Acceptance {
|
||||
id: 0,
|
||||
check: "SS-1".into(),
|
||||
subject: String::new(),
|
||||
accepted_value: serde_json::json!(true),
|
||||
note: "Old clients on the LAN; closed by 2027.".into(),
|
||||
accepted_by: String::new(),
|
||||
accepted_at: 0,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_note_is_required() {
|
||||
assert!(acceptance().validate().is_ok());
|
||||
let blank = Acceptance {
|
||||
note: " ".into(),
|
||||
..acceptance()
|
||||
};
|
||||
assert_eq!(blank.validate().unwrap_err().property, "note");
|
||||
let long = Acceptance {
|
||||
note: "x".repeat(501),
|
||||
..acceptance()
|
||||
};
|
||||
assert_eq!(long.validate().unwrap_err().property, "note");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_named_checks() {
|
||||
for bad in ["", "SS-0", "SS-41", "ss-1", "SS-x", "1"] {
|
||||
let a = Acceptance {
|
||||
check: bad.into(),
|
||||
..acceptance()
|
||||
};
|
||||
assert_eq!(a.validate().unwrap_err().property, "check", "{bad}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subject_and_value_are_bounded() {
|
||||
let a = Acceptance {
|
||||
subject: "d".repeat(256),
|
||||
..acceptance()
|
||||
};
|
||||
assert_eq!(a.validate().unwrap_err().property, "subject");
|
||||
let a = Acceptance {
|
||||
accepted_value: serde_json::json!("v".repeat(4096)),
|
||||
..acceptance()
|
||||
};
|
||||
assert_eq!(a.validate().unwrap_err().property, "acceptedValue");
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@
|
||||
//! ships. The legacy-protocols switch is INBUXA's own design, specified in
|
||||
//! `legacy-protocols.md`.
|
||||
|
||||
pub mod acceptance;
|
||||
pub mod legacy_use;
|
||||
pub mod log_files;
|
||||
pub mod listeners;
|
||||
|
||||
@@ -120,6 +120,40 @@ impl ManagementApi for Server {
|
||||
jmap::inbuxa::explanation::question(self, &access_token, &subject).await?;
|
||||
Ok(explain_stream(self.clone(), access_token, question, in_flight))
|
||||
}
|
||||
// inbuxa: try a saved directory before anything signs in through it
|
||||
"directory" if is_post && path.get(1).copied() == Some("test") => {
|
||||
let (_in_flight, access_token) = self.authenticate_headers(req, session).await?;
|
||||
jmap::inbuxa::directory_test::assert_allowed(&access_token)?;
|
||||
let request = body
|
||||
.as_deref()
|
||||
.and_then(|body| serde_json::from_slice::<serde_json::Value>(body).ok())
|
||||
.unwrap_or_default();
|
||||
let answer = jmap::inbuxa::directory_test::test(self, &request).await?;
|
||||
Ok(JsonResponse::new(answer).no_cache().into_http_response())
|
||||
}
|
||||
// inbuxa: send one sample event to a saved webhook
|
||||
"webhook" if is_post && path.get(1).copied() == Some("test") => {
|
||||
let (_in_flight, access_token) = self.authenticate_headers(req, session).await?;
|
||||
jmap::inbuxa::webhook_test::assert_allowed(&access_token)?;
|
||||
let request = body
|
||||
.as_deref()
|
||||
.and_then(|body| serde_json::from_slice::<serde_json::Value>(body).ok())
|
||||
.unwrap_or_default();
|
||||
let answer = jmap::inbuxa::webhook_test::test(self, &request).await?;
|
||||
Ok(JsonResponse::new(answer).no_cache().into_http_response())
|
||||
}
|
||||
// inbuxa: whether the outside world reaches each node's ports
|
||||
"ports" if path.get(1).copied() == Some("check") => {
|
||||
let (_in_flight, access_token) = self.authenticate_headers(req, session).await?;
|
||||
if access_token.tenant_id().is_some() {
|
||||
return Err(trc::JmapEvent::Forbidden
|
||||
.into_err()
|
||||
.details("Port checks are for server-level administrators."));
|
||||
}
|
||||
access_token.enforce_permission(Permission::SysNetworkListenerGet)?;
|
||||
let answer = common::reachability::report(self).await?;
|
||||
Ok(JsonResponse::new(answer).no_cache().into_http_response())
|
||||
}
|
||||
"account" => {
|
||||
// Authenticate request
|
||||
let (_in_flight, access_token) = self.authenticate_headers(req, session).await?;
|
||||
|
||||
@@ -47,6 +47,18 @@ struct SetErrorInner<P: Property> {
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
#[serde(rename = "validationErrors")]
|
||||
validation_errors: Vec<ValidationError>,
|
||||
|
||||
// inbuxa: DLP (dlp-and-mail-flow-rules spec, §2.5): each rule that
|
||||
// warned or blocked, with its notice
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
rules: Vec<DlpRule>,
|
||||
}
|
||||
|
||||
/// inbuxa: a DLP rule named in an `inbuxa:dlpWarning` or `inbuxa:dlpBlocked`.
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct DlpRule {
|
||||
pub name: String,
|
||||
pub notice: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -127,6 +139,12 @@ pub enum SetErrorType {
|
||||
// inbuxa: a create that couldn't run (ai-explain spec: busy, timeout, …)
|
||||
#[serde(rename = "serverFail")]
|
||||
ServerFail,
|
||||
// inbuxa: DLP (dlp-and-mail-flow-rules spec, §2.5): a warning the
|
||||
// sender may answer with inbuxa:dlpOverride, and a block
|
||||
#[serde(rename = "inbuxa:dlpWarning")]
|
||||
DlpWarning,
|
||||
#[serde(rename = "inbuxa:dlpBlocked")]
|
||||
DlpBlocked,
|
||||
}
|
||||
|
||||
impl SetErrorType {
|
||||
@@ -166,6 +184,8 @@ impl SetErrorType {
|
||||
SetErrorType::PrimaryKeyViolation => "primaryKeyViolation",
|
||||
SetErrorType::ValidationFailed => "validationFailed",
|
||||
SetErrorType::ServerFail => "serverFail",
|
||||
SetErrorType::DlpWarning => "inbuxa:dlpWarning",
|
||||
SetErrorType::DlpBlocked => "inbuxa:dlpBlocked",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -180,9 +200,16 @@ impl<T: Property> SetError<T> {
|
||||
object_id: None,
|
||||
linked_objects: Vec::new(),
|
||||
validation_errors: Vec::new(),
|
||||
rules: Vec::new(),
|
||||
}))
|
||||
}
|
||||
|
||||
/// inbuxa: the DLP rules behind a warning or block.
|
||||
pub fn with_dlp_rules(mut self, rules: Vec<DlpRule>) -> Self {
|
||||
self.0.rules = rules;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_description(mut self, description: impl Into<Cow<'static, str>>) -> Self {
|
||||
self.0.description = description.into().into();
|
||||
self
|
||||
@@ -353,6 +380,7 @@ impl From<PatchError> for SetError<registry::schema::properties::Property> {
|
||||
object_id: None,
|
||||
linked_objects: Vec::new(),
|
||||
validation_errors: Vec::new(),
|
||||
rules: Vec::new(),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use crate::{
|
||||
@@ -40,6 +42,12 @@ pub enum EmailSubmissionProperty {
|
||||
Displayed,
|
||||
DsnBlobIds,
|
||||
MdnBlobIds,
|
||||
// inbuxa: DLP (dlp-and-mail-flow-rules spec, §2.5): `{"reason": ...}`
|
||||
// to send despite a warning
|
||||
DlpOverride,
|
||||
// inbuxa: in a create's response, true when DLP held the message for
|
||||
// review (§2.6)
|
||||
DlpHeld,
|
||||
|
||||
Pointer(JsonPointer<EmailSubmissionProperty>),
|
||||
}
|
||||
@@ -90,6 +98,8 @@ impl Property for EmailSubmissionProperty {
|
||||
EmailSubmissionProperty::Id => "id",
|
||||
EmailSubmissionProperty::IdentityId => "identityId",
|
||||
EmailSubmissionProperty::MdnBlobIds => "mdnBlobIds",
|
||||
EmailSubmissionProperty::DlpOverride => "inbuxa:dlpOverride",
|
||||
EmailSubmissionProperty::DlpHeld => "inbuxa:held",
|
||||
EmailSubmissionProperty::SendAt => "sendAt",
|
||||
EmailSubmissionProperty::ThreadId => "threadId",
|
||||
EmailSubmissionProperty::UndoStatus => "undoStatus",
|
||||
@@ -181,6 +191,8 @@ impl EmailSubmissionProperty {
|
||||
"displayed" => EmailSubmissionProperty::Displayed,
|
||||
"dsnBlobIds" => EmailSubmissionProperty::DsnBlobIds,
|
||||
"mdnBlobIds" => EmailSubmissionProperty::MdnBlobIds,
|
||||
"inbuxa:dlpOverride" => EmailSubmissionProperty::DlpOverride,
|
||||
"inbuxa:held" => EmailSubmissionProperty::DlpHeld,
|
||||
)
|
||||
.or_else(|| {
|
||||
if allow_patch && value.contains('/') {
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! `inbuxa:DlpSettings/get` and `/set` under `urn:inbuxa:jmap`: the DLP
|
||||
//! settings singleton (dlp-and-mail-flow-rules spec, §2.6): how many days
|
||||
//! held mail waits for a reviewer before it goes back to the sender.
|
||||
|
||||
use crate::object::{AnyId, JmapObject, JmapObjectId};
|
||||
use jmap_tools::{Element, Key, Property};
|
||||
use std::{borrow::Cow, str::FromStr};
|
||||
use types::id::Id;
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct DlpSettings;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum DlpSettingsProperty {
|
||||
Id,
|
||||
KeepHeldDays,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum DlpSettingsValue {
|
||||
Id(Id),
|
||||
}
|
||||
|
||||
impl Property for DlpSettingsProperty {
|
||||
fn try_parse(_: Option<&Key<'_, Self>>, value: &str) -> Option<Self> {
|
||||
DlpSettingsProperty::parse(value)
|
||||
}
|
||||
|
||||
fn to_cow(&self) -> Cow<'static, str> {
|
||||
match self {
|
||||
DlpSettingsProperty::Id => "id",
|
||||
DlpSettingsProperty::KeepHeldDays => "keepHeldDays",
|
||||
}
|
||||
.into()
|
||||
}
|
||||
}
|
||||
|
||||
impl DlpSettingsProperty {
|
||||
fn parse(value: &str) -> Option<Self> {
|
||||
hashify::tiny_map!(value.as_bytes(),
|
||||
b"id" => DlpSettingsProperty::Id,
|
||||
b"keepHeldDays" => DlpSettingsProperty::KeepHeldDays,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for DlpSettingsProperty {
|
||||
type Err = ();
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
DlpSettingsProperty::parse(s).ok_or(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Element for DlpSettingsValue {
|
||||
type Property = DlpSettingsProperty;
|
||||
|
||||
fn try_parse<P>(key: &Key<'_, Self::Property>, value: &str) -> Option<Self> {
|
||||
match key {
|
||||
Key::Property(DlpSettingsProperty::Id) => {
|
||||
Id::from_str(value).ok().map(DlpSettingsValue::Id)
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn to_cow(&self) -> Cow<'static, str> {
|
||||
match self {
|
||||
DlpSettingsValue::Id(id) => id.to_string().into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl JmapObject for DlpSettings {
|
||||
type Property = DlpSettingsProperty;
|
||||
|
||||
type Element = DlpSettingsValue;
|
||||
|
||||
type Id = Id;
|
||||
|
||||
type Filter = ();
|
||||
|
||||
type Comparator = ();
|
||||
|
||||
type GetArguments = ();
|
||||
|
||||
type SetArguments<'de> = ();
|
||||
|
||||
type QueryArguments = ();
|
||||
|
||||
type CopyArguments = ();
|
||||
|
||||
type ParseArguments = ();
|
||||
|
||||
const ID_PROPERTY: Self::Property = DlpSettingsProperty::Id;
|
||||
}
|
||||
|
||||
impl From<Id> for DlpSettingsValue {
|
||||
fn from(id: Id) -> Self {
|
||||
DlpSettingsValue::Id(id)
|
||||
}
|
||||
}
|
||||
|
||||
impl JmapObjectId for DlpSettingsValue {
|
||||
fn as_id(&self) -> Option<Id> {
|
||||
match self {
|
||||
DlpSettingsValue::Id(id) => Some(*id),
|
||||
}
|
||||
}
|
||||
|
||||
fn as_any_id(&self) -> Option<AnyId> {
|
||||
match self {
|
||||
DlpSettingsValue::Id(id) => Some(AnyId::Id(*id)),
|
||||
}
|
||||
}
|
||||
|
||||
fn as_id_ref(&self) -> Option<&str> {
|
||||
None
|
||||
}
|
||||
|
||||
fn try_set_id(&mut self, new_id: AnyId) -> bool {
|
||||
if let AnyId::Id(id) = new_id {
|
||||
*self = DlpSettingsValue::Id(id);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl JmapObjectId for DlpSettingsProperty {
|
||||
fn as_id(&self) -> Option<Id> {
|
||||
None
|
||||
}
|
||||
|
||||
fn as_any_id(&self) -> Option<AnyId> {
|
||||
None
|
||||
}
|
||||
|
||||
fn as_id_ref(&self) -> Option<&str> {
|
||||
None
|
||||
}
|
||||
|
||||
fn try_set_id(&mut self, _: AnyId) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! `inbuxa:HeldMessage/get` and `/set` under `urn:inbuxa:jmap`: mail held
|
||||
//! for review (dlp-and-mail-flow-rules spec, §2.6). Get lists it; `preview`
|
||||
//! (the text, only when asked for) is recorded as access to someone's mail.
|
||||
//! Set only updates: `{"decision": "release"}`, or `"reject"` with an
|
||||
//! optional `note` for the sender. The call's `reason` goes into the audit
|
||||
//! log and is required.
|
||||
|
||||
use crate::{
|
||||
object::{AnyId, JmapObject, JmapObjectId},
|
||||
request::deserialize::DeserializeArguments,
|
||||
};
|
||||
use jmap_tools::{Element, Key, Property};
|
||||
use std::{borrow::Cow, str::FromStr};
|
||||
use types::id::Id;
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct HeldMessage;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum HeldMessageProperty {
|
||||
Id,
|
||||
Sender,
|
||||
Recipients,
|
||||
Subject,
|
||||
Size,
|
||||
Rules,
|
||||
Counts,
|
||||
HeldAt,
|
||||
ExpiresAt,
|
||||
Preview,
|
||||
Decision,
|
||||
Note,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum HeldMessageValue {
|
||||
Id(Id),
|
||||
}
|
||||
|
||||
impl Property for HeldMessageProperty {
|
||||
fn try_parse(parent: Option<&Key<'_, Self>>, value: &str) -> Option<Self> {
|
||||
// Keys inside rules and counts stay plain keys
|
||||
match parent {
|
||||
None => HeldMessageProperty::parse(value),
|
||||
Some(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn to_cow(&self) -> Cow<'static, str> {
|
||||
match self {
|
||||
HeldMessageProperty::Id => "id",
|
||||
HeldMessageProperty::Sender => "sender",
|
||||
HeldMessageProperty::Recipients => "recipients",
|
||||
HeldMessageProperty::Subject => "subject",
|
||||
HeldMessageProperty::Size => "size",
|
||||
HeldMessageProperty::Rules => "rules",
|
||||
HeldMessageProperty::Counts => "counts",
|
||||
HeldMessageProperty::HeldAt => "heldAt",
|
||||
HeldMessageProperty::ExpiresAt => "expiresAt",
|
||||
HeldMessageProperty::Preview => "preview",
|
||||
HeldMessageProperty::Decision => "decision",
|
||||
HeldMessageProperty::Note => "note",
|
||||
}
|
||||
.into()
|
||||
}
|
||||
}
|
||||
|
||||
impl HeldMessageProperty {
|
||||
fn parse(value: &str) -> Option<Self> {
|
||||
hashify::tiny_map!(value.as_bytes(),
|
||||
b"id" => HeldMessageProperty::Id,
|
||||
b"sender" => HeldMessageProperty::Sender,
|
||||
b"recipients" => HeldMessageProperty::Recipients,
|
||||
b"subject" => HeldMessageProperty::Subject,
|
||||
b"size" => HeldMessageProperty::Size,
|
||||
b"rules" => HeldMessageProperty::Rules,
|
||||
b"counts" => HeldMessageProperty::Counts,
|
||||
b"heldAt" => HeldMessageProperty::HeldAt,
|
||||
b"expiresAt" => HeldMessageProperty::ExpiresAt,
|
||||
b"preview" => HeldMessageProperty::Preview,
|
||||
b"decision" => HeldMessageProperty::Decision,
|
||||
b"note" => HeldMessageProperty::Note,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for HeldMessageProperty {
|
||||
type Err = ();
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
HeldMessageProperty::parse(s).ok_or(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Element for HeldMessageValue {
|
||||
type Property = HeldMessageProperty;
|
||||
|
||||
fn try_parse<P>(key: &Key<'_, Self::Property>, value: &str) -> Option<Self> {
|
||||
match key {
|
||||
Key::Property(HeldMessageProperty::Id) => {
|
||||
Id::from_str(value).ok().map(HeldMessageValue::Id)
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn to_cow(&self) -> Cow<'static, str> {
|
||||
match self {
|
||||
HeldMessageValue::Id(id) => id.to_string().into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The set call's own argument: why, for the audit log (required).
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct HeldMessageSetArguments {
|
||||
pub reason: Option<String>,
|
||||
}
|
||||
|
||||
impl<'de> DeserializeArguments<'de> for HeldMessageSetArguments {
|
||||
fn deserialize_argument<A>(&mut self, key: &str, map: &mut A) -> Result<(), A::Error>
|
||||
where
|
||||
A: serde::de::MapAccess<'de>,
|
||||
{
|
||||
if key == "reason" {
|
||||
self.reason = map.next_value()?;
|
||||
} else {
|
||||
let _ = map.next_value::<serde::de::IgnoredAny>()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl JmapObject for HeldMessage {
|
||||
type Property = HeldMessageProperty;
|
||||
|
||||
type Element = HeldMessageValue;
|
||||
|
||||
type Id = Id;
|
||||
|
||||
type Filter = ();
|
||||
|
||||
type Comparator = ();
|
||||
|
||||
type GetArguments = ();
|
||||
|
||||
type SetArguments<'de> = HeldMessageSetArguments;
|
||||
|
||||
type QueryArguments = ();
|
||||
|
||||
type CopyArguments = ();
|
||||
|
||||
type ParseArguments = ();
|
||||
|
||||
const ID_PROPERTY: Self::Property = HeldMessageProperty::Id;
|
||||
}
|
||||
|
||||
impl From<Id> for HeldMessageValue {
|
||||
fn from(id: Id) -> Self {
|
||||
HeldMessageValue::Id(id)
|
||||
}
|
||||
}
|
||||
|
||||
impl JmapObjectId for HeldMessageValue {
|
||||
fn as_id(&self) -> Option<Id> {
|
||||
match self {
|
||||
HeldMessageValue::Id(id) => Some(*id),
|
||||
}
|
||||
}
|
||||
|
||||
fn as_any_id(&self) -> Option<AnyId> {
|
||||
match self {
|
||||
HeldMessageValue::Id(id) => Some(AnyId::Id(*id)),
|
||||
}
|
||||
}
|
||||
|
||||
fn as_id_ref(&self) -> Option<&str> {
|
||||
None
|
||||
}
|
||||
|
||||
fn try_set_id(&mut self, new_id: AnyId) -> bool {
|
||||
if let AnyId::Id(id) = new_id {
|
||||
*self = HeldMessageValue::Id(id);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl JmapObjectId for HeldMessageProperty {
|
||||
fn as_id(&self) -> Option<Id> {
|
||||
None
|
||||
}
|
||||
|
||||
fn as_any_id(&self) -> Option<AnyId> {
|
||||
None
|
||||
}
|
||||
|
||||
fn as_id_ref(&self) -> Option<&str> {
|
||||
None
|
||||
}
|
||||
|
||||
fn try_set_id(&mut self, _: AnyId) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! `inbuxa:Journal/get` and `/set` under `urn:inbuxa:jmap`: journals
|
||||
//! (journaling spec, JR-9, JR-12). What a journal has taken stays when the
|
||||
//! journal changes or goes; each entry keeps its own retention.
|
||||
|
||||
use crate::{
|
||||
object::{AnyId, JmapObject, JmapObjectId},
|
||||
request::deserialize::DeserializeArguments,
|
||||
};
|
||||
use jmap_tools::{Element, Key, Property};
|
||||
use std::{borrow::Cow, str::FromStr};
|
||||
use types::id::Id;
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct Journal;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum JournalProperty {
|
||||
Id,
|
||||
Name,
|
||||
Description,
|
||||
Enabled,
|
||||
/// `outgoing`, `incoming`, `internal` or `any`.
|
||||
Direction,
|
||||
/// Everyone, or chosen accounts, groups, domains and tenants.
|
||||
Scope,
|
||||
/// How long an entry is kept; each keeps what it was written with.
|
||||
RetentionDays,
|
||||
CreatedBy,
|
||||
CreatedAt,
|
||||
UpdatedAt,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum JournalValue {
|
||||
Id(Id),
|
||||
}
|
||||
|
||||
impl Property for JournalProperty {
|
||||
fn try_parse(parent: Option<&Key<'_, Self>>, value: &str) -> Option<Self> {
|
||||
// Keys inside the scope stay plain keys
|
||||
match parent {
|
||||
None => JournalProperty::parse(value),
|
||||
Some(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn to_cow(&self) -> Cow<'static, str> {
|
||||
match self {
|
||||
JournalProperty::Id => "id",
|
||||
JournalProperty::Name => "name",
|
||||
JournalProperty::Description => "description",
|
||||
JournalProperty::Enabled => "enabled",
|
||||
JournalProperty::Direction => "direction",
|
||||
JournalProperty::Scope => "scope",
|
||||
JournalProperty::RetentionDays => "retentionDays",
|
||||
JournalProperty::CreatedBy => "createdBy",
|
||||
JournalProperty::CreatedAt => "createdAt",
|
||||
JournalProperty::UpdatedAt => "updatedAt",
|
||||
}
|
||||
.into()
|
||||
}
|
||||
}
|
||||
|
||||
impl JournalProperty {
|
||||
fn parse(value: &str) -> Option<Self> {
|
||||
hashify::tiny_map!(value.as_bytes(),
|
||||
b"id" => JournalProperty::Id,
|
||||
b"name" => JournalProperty::Name,
|
||||
b"description" => JournalProperty::Description,
|
||||
b"enabled" => JournalProperty::Enabled,
|
||||
b"direction" => JournalProperty::Direction,
|
||||
b"scope" => JournalProperty::Scope,
|
||||
b"retentionDays" => JournalProperty::RetentionDays,
|
||||
b"createdBy" => JournalProperty::CreatedBy,
|
||||
b"createdAt" => JournalProperty::CreatedAt,
|
||||
b"updatedAt" => JournalProperty::UpdatedAt,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for JournalProperty {
|
||||
type Err = ();
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
JournalProperty::parse(s).ok_or(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Element for JournalValue {
|
||||
type Property = JournalProperty;
|
||||
|
||||
fn try_parse<P>(key: &Key<'_, Self::Property>, value: &str) -> Option<Self> {
|
||||
match key {
|
||||
Key::Property(JournalProperty::Id) => Id::from_str(value).ok().map(JournalValue::Id),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn to_cow(&self) -> Cow<'static, str> {
|
||||
match self {
|
||||
JournalValue::Id(id) => id.to_string().into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The set call's own argument: why, for the audit log.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct JournalSetArguments {
|
||||
pub reason: Option<String>,
|
||||
}
|
||||
|
||||
impl<'de> DeserializeArguments<'de> for JournalSetArguments {
|
||||
fn deserialize_argument<A>(&mut self, key: &str, map: &mut A) -> Result<(), A::Error>
|
||||
where
|
||||
A: serde::de::MapAccess<'de>,
|
||||
{
|
||||
if key == "reason" {
|
||||
self.reason = map.next_value()?;
|
||||
} else {
|
||||
let _ = map.next_value::<serde::de::IgnoredAny>()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl JmapObject for Journal {
|
||||
type Property = JournalProperty;
|
||||
|
||||
type Element = JournalValue;
|
||||
|
||||
type Id = Id;
|
||||
|
||||
type Filter = ();
|
||||
|
||||
type Comparator = ();
|
||||
|
||||
type GetArguments = ();
|
||||
|
||||
type SetArguments<'de> = JournalSetArguments;
|
||||
|
||||
type QueryArguments = ();
|
||||
|
||||
type CopyArguments = ();
|
||||
|
||||
type ParseArguments = ();
|
||||
|
||||
const ID_PROPERTY: Self::Property = JournalProperty::Id;
|
||||
}
|
||||
|
||||
impl From<Id> for JournalValue {
|
||||
fn from(id: Id) -> Self {
|
||||
JournalValue::Id(id)
|
||||
}
|
||||
}
|
||||
|
||||
impl JmapObjectId for JournalValue {
|
||||
fn as_id(&self) -> Option<Id> {
|
||||
match self {
|
||||
JournalValue::Id(id) => Some(*id),
|
||||
}
|
||||
}
|
||||
|
||||
fn as_any_id(&self) -> Option<AnyId> {
|
||||
match self {
|
||||
JournalValue::Id(id) => Some(AnyId::Id(*id)),
|
||||
}
|
||||
}
|
||||
|
||||
fn as_id_ref(&self) -> Option<&str> {
|
||||
None
|
||||
}
|
||||
|
||||
fn try_set_id(&mut self, new_id: AnyId) -> bool {
|
||||
if let AnyId::Id(id) = new_id {
|
||||
*self = JournalValue::Id(id);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl JmapObjectId for JournalProperty {
|
||||
fn as_id(&self) -> Option<Id> {
|
||||
None
|
||||
}
|
||||
|
||||
fn as_any_id(&self) -> Option<AnyId> {
|
||||
None
|
||||
}
|
||||
|
||||
fn as_id_ref(&self) -> Option<&str> {
|
||||
None
|
||||
}
|
||||
|
||||
fn try_set_id(&mut self, _: AnyId) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! `inbuxa:MailRule/get` and `/set` under `urn:inbuxa:jmap`: mail flow rules
|
||||
//! and DLP rules (dlp-and-mail-flow-rules spec, §2.2). `kind` says which,
|
||||
//! and which permissions reach it. The set call's `reason` argument, if
|
||||
//! given, goes into the audit log with the change.
|
||||
|
||||
use crate::{
|
||||
object::{AnyId, JmapObject, JmapObjectId},
|
||||
request::deserialize::DeserializeArguments,
|
||||
};
|
||||
use jmap_tools::{Element, Key, Property};
|
||||
use std::{borrow::Cow, str::FromStr};
|
||||
use types::id::Id;
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct MailRule;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum MailRuleProperty {
|
||||
Id,
|
||||
Name,
|
||||
Description,
|
||||
/// `dlp` or `transport`.
|
||||
Kind,
|
||||
Enabled,
|
||||
/// Lower runs first.
|
||||
Priority,
|
||||
/// `outgoing`, `incoming` or `any`.
|
||||
Direction,
|
||||
Conditions,
|
||||
Exceptions,
|
||||
Actions,
|
||||
StopProcessing,
|
||||
CreatedBy,
|
||||
CreatedAt,
|
||||
UpdatedAt,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum MailRuleValue {
|
||||
Id(Id),
|
||||
}
|
||||
|
||||
impl Property for MailRuleProperty {
|
||||
fn try_parse(parent: Option<&Key<'_, Self>>, value: &str) -> Option<Self> {
|
||||
// Keys inside conditions and actions stay plain keys
|
||||
match parent {
|
||||
None => MailRuleProperty::parse(value),
|
||||
Some(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn to_cow(&self) -> Cow<'static, str> {
|
||||
match self {
|
||||
MailRuleProperty::Id => "id",
|
||||
MailRuleProperty::Name => "name",
|
||||
MailRuleProperty::Description => "description",
|
||||
MailRuleProperty::Kind => "kind",
|
||||
MailRuleProperty::Enabled => "enabled",
|
||||
MailRuleProperty::Priority => "priority",
|
||||
MailRuleProperty::Direction => "direction",
|
||||
MailRuleProperty::Conditions => "conditions",
|
||||
MailRuleProperty::Exceptions => "exceptions",
|
||||
MailRuleProperty::Actions => "actions",
|
||||
MailRuleProperty::StopProcessing => "stopProcessing",
|
||||
MailRuleProperty::CreatedBy => "createdBy",
|
||||
MailRuleProperty::CreatedAt => "createdAt",
|
||||
MailRuleProperty::UpdatedAt => "updatedAt",
|
||||
}
|
||||
.into()
|
||||
}
|
||||
}
|
||||
|
||||
impl MailRuleProperty {
|
||||
fn parse(value: &str) -> Option<Self> {
|
||||
hashify::tiny_map!(value.as_bytes(),
|
||||
b"id" => MailRuleProperty::Id,
|
||||
b"name" => MailRuleProperty::Name,
|
||||
b"description" => MailRuleProperty::Description,
|
||||
b"kind" => MailRuleProperty::Kind,
|
||||
b"enabled" => MailRuleProperty::Enabled,
|
||||
b"priority" => MailRuleProperty::Priority,
|
||||
b"direction" => MailRuleProperty::Direction,
|
||||
b"conditions" => MailRuleProperty::Conditions,
|
||||
b"exceptions" => MailRuleProperty::Exceptions,
|
||||
b"actions" => MailRuleProperty::Actions,
|
||||
b"stopProcessing" => MailRuleProperty::StopProcessing,
|
||||
b"createdBy" => MailRuleProperty::CreatedBy,
|
||||
b"createdAt" => MailRuleProperty::CreatedAt,
|
||||
b"updatedAt" => MailRuleProperty::UpdatedAt,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for MailRuleProperty {
|
||||
type Err = ();
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
MailRuleProperty::parse(s).ok_or(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Element for MailRuleValue {
|
||||
type Property = MailRuleProperty;
|
||||
|
||||
fn try_parse<P>(key: &Key<'_, Self::Property>, value: &str) -> Option<Self> {
|
||||
match key {
|
||||
Key::Property(MailRuleProperty::Id) => Id::from_str(value).ok().map(MailRuleValue::Id),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn to_cow(&self) -> Cow<'static, str> {
|
||||
match self {
|
||||
MailRuleValue::Id(id) => id.to_string().into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The set call's own argument: why, for the audit log.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct MailRuleSetArguments {
|
||||
pub reason: Option<String>,
|
||||
}
|
||||
|
||||
impl<'de> DeserializeArguments<'de> for MailRuleSetArguments {
|
||||
fn deserialize_argument<A>(&mut self, key: &str, map: &mut A) -> Result<(), A::Error>
|
||||
where
|
||||
A: serde::de::MapAccess<'de>,
|
||||
{
|
||||
if key == "reason" {
|
||||
self.reason = map.next_value()?;
|
||||
} else {
|
||||
let _ = map.next_value::<serde::de::IgnoredAny>()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl JmapObject for MailRule {
|
||||
type Property = MailRuleProperty;
|
||||
|
||||
type Element = MailRuleValue;
|
||||
|
||||
type Id = Id;
|
||||
|
||||
type Filter = ();
|
||||
|
||||
type Comparator = ();
|
||||
|
||||
type GetArguments = ();
|
||||
|
||||
type SetArguments<'de> = MailRuleSetArguments;
|
||||
|
||||
type QueryArguments = ();
|
||||
|
||||
type CopyArguments = ();
|
||||
|
||||
type ParseArguments = ();
|
||||
|
||||
const ID_PROPERTY: Self::Property = MailRuleProperty::Id;
|
||||
}
|
||||
|
||||
impl From<Id> for MailRuleValue {
|
||||
fn from(id: Id) -> Self {
|
||||
MailRuleValue::Id(id)
|
||||
}
|
||||
}
|
||||
|
||||
impl JmapObjectId for MailRuleValue {
|
||||
fn as_id(&self) -> Option<Id> {
|
||||
match self {
|
||||
MailRuleValue::Id(id) => Some(*id),
|
||||
}
|
||||
}
|
||||
|
||||
fn as_any_id(&self) -> Option<AnyId> {
|
||||
match self {
|
||||
MailRuleValue::Id(id) => Some(AnyId::Id(*id)),
|
||||
}
|
||||
}
|
||||
|
||||
fn as_id_ref(&self) -> Option<&str> {
|
||||
None
|
||||
}
|
||||
|
||||
fn try_set_id(&mut self, new_id: AnyId) -> bool {
|
||||
if let AnyId::Id(id) = new_id {
|
||||
*self = MailRuleValue::Id(id);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl JmapObjectId for MailRuleProperty {
|
||||
fn as_id(&self) -> Option<Id> {
|
||||
None
|
||||
}
|
||||
|
||||
fn as_any_id(&self) -> Option<AnyId> {
|
||||
None
|
||||
}
|
||||
|
||||
fn as_id_ref(&self) -> Option<&str> {
|
||||
None
|
||||
}
|
||||
|
||||
fn try_set_id(&mut self, _: AnyId) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! `inbuxa:SecurityAcceptance/get` and `/set` under `urn:inbuxa:jmap`: the
|
||||
//! security to-do items an administrator accepted, with why (security
|
||||
//! to-do list spec, SS-23 to SS-26). Created and destroyed, never updated.
|
||||
|
||||
use crate::object::{AnyId, JmapObject, JmapObjectId};
|
||||
use jmap_tools::{Element, Key, Property};
|
||||
use std::{borrow::Cow, str::FromStr};
|
||||
use types::id::Id;
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct SecurityAcceptance;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum SecurityAcceptanceProperty {
|
||||
Id,
|
||||
/// `SS-1` to `SS-18`.
|
||||
Check,
|
||||
Subject,
|
||||
AcceptedValue,
|
||||
Note,
|
||||
AcceptedBy,
|
||||
AcceptedAt,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum SecurityAcceptanceValue {
|
||||
Id(Id),
|
||||
}
|
||||
|
||||
impl Property for SecurityAcceptanceProperty {
|
||||
fn try_parse(parent: Option<&Key<'_, Self>>, value: &str) -> Option<Self> {
|
||||
// Keys inside acceptedValue stay plain keys
|
||||
match parent {
|
||||
None => SecurityAcceptanceProperty::parse(value),
|
||||
Some(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn to_cow(&self) -> Cow<'static, str> {
|
||||
match self {
|
||||
SecurityAcceptanceProperty::Id => "id",
|
||||
SecurityAcceptanceProperty::Check => "check",
|
||||
SecurityAcceptanceProperty::Subject => "subject",
|
||||
SecurityAcceptanceProperty::AcceptedValue => "acceptedValue",
|
||||
SecurityAcceptanceProperty::Note => "note",
|
||||
SecurityAcceptanceProperty::AcceptedBy => "acceptedBy",
|
||||
SecurityAcceptanceProperty::AcceptedAt => "acceptedAt",
|
||||
}
|
||||
.into()
|
||||
}
|
||||
}
|
||||
|
||||
impl SecurityAcceptanceProperty {
|
||||
fn parse(value: &str) -> Option<Self> {
|
||||
hashify::tiny_map!(value.as_bytes(),
|
||||
b"id" => SecurityAcceptanceProperty::Id,
|
||||
b"check" => SecurityAcceptanceProperty::Check,
|
||||
b"subject" => SecurityAcceptanceProperty::Subject,
|
||||
b"acceptedValue" => SecurityAcceptanceProperty::AcceptedValue,
|
||||
b"note" => SecurityAcceptanceProperty::Note,
|
||||
b"acceptedBy" => SecurityAcceptanceProperty::AcceptedBy,
|
||||
b"acceptedAt" => SecurityAcceptanceProperty::AcceptedAt,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for SecurityAcceptanceProperty {
|
||||
type Err = ();
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
SecurityAcceptanceProperty::parse(s).ok_or(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Element for SecurityAcceptanceValue {
|
||||
type Property = SecurityAcceptanceProperty;
|
||||
|
||||
fn try_parse<P>(key: &Key<'_, Self::Property>, value: &str) -> Option<Self> {
|
||||
match key {
|
||||
Key::Property(SecurityAcceptanceProperty::Id) => {
|
||||
Id::from_str(value).ok().map(SecurityAcceptanceValue::Id)
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn to_cow(&self) -> Cow<'static, str> {
|
||||
match self {
|
||||
SecurityAcceptanceValue::Id(id) => id.to_string().into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl JmapObject for SecurityAcceptance {
|
||||
type Property = SecurityAcceptanceProperty;
|
||||
|
||||
type Element = SecurityAcceptanceValue;
|
||||
|
||||
type Id = Id;
|
||||
|
||||
type Filter = ();
|
||||
|
||||
type Comparator = ();
|
||||
|
||||
type GetArguments = ();
|
||||
|
||||
type SetArguments<'de> = ();
|
||||
|
||||
type QueryArguments = ();
|
||||
|
||||
type CopyArguments = ();
|
||||
|
||||
type ParseArguments = ();
|
||||
|
||||
const ID_PROPERTY: Self::Property = SecurityAcceptanceProperty::Id;
|
||||
}
|
||||
|
||||
impl From<Id> for SecurityAcceptanceValue {
|
||||
fn from(id: Id) -> Self {
|
||||
SecurityAcceptanceValue::Id(id)
|
||||
}
|
||||
}
|
||||
|
||||
impl JmapObjectId for SecurityAcceptanceValue {
|
||||
fn as_id(&self) -> Option<Id> {
|
||||
match self {
|
||||
SecurityAcceptanceValue::Id(id) => Some(*id),
|
||||
}
|
||||
}
|
||||
|
||||
fn as_any_id(&self) -> Option<AnyId> {
|
||||
match self {
|
||||
SecurityAcceptanceValue::Id(id) => Some(AnyId::Id(*id)),
|
||||
}
|
||||
}
|
||||
|
||||
fn as_id_ref(&self) -> Option<&str> {
|
||||
None
|
||||
}
|
||||
|
||||
fn try_set_id(&mut self, new_id: AnyId) -> bool {
|
||||
if let AnyId::Id(id) = new_id {
|
||||
*self = SecurityAcceptanceValue::Id(id);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl JmapObjectId for SecurityAcceptanceProperty {
|
||||
fn as_id(&self) -> Option<Id> {
|
||||
None
|
||||
}
|
||||
|
||||
fn as_any_id(&self) -> Option<AnyId> {
|
||||
None
|
||||
}
|
||||
|
||||
fn as_id_ref(&self) -> Option<&str> {
|
||||
None
|
||||
}
|
||||
|
||||
fn try_set_id(&mut self, _: AnyId) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
@@ -24,10 +24,15 @@ pub mod fastmail_masked_email; // inbuxa: masked email
|
||||
pub mod inbuxa_account_lock; // inbuxa: account lock with delegation
|
||||
pub mod inbuxa_ai_limits; // inbuxa: AI spam classification
|
||||
pub mod inbuxa_log_settings; // inbuxa: personal-data catalog, D1
|
||||
pub mod inbuxa_dlp_settings; // inbuxa: DLP settings
|
||||
pub mod inbuxa_data_inventory; // inbuxa: personal-data catalog
|
||||
pub mod inbuxa_inventory_snapshot; // inbuxa: personal-data catalog
|
||||
pub mod inbuxa_audit; // inbuxa: the audit log
|
||||
pub mod inbuxa_legal_hold; // inbuxa: legal hold
|
||||
pub mod inbuxa_mail_rule; // inbuxa: DLP and mail flow rules
|
||||
pub mod inbuxa_security_acceptance; // inbuxa: accepted security to-do items
|
||||
pub mod inbuxa_journal; // inbuxa: journaling
|
||||
pub mod inbuxa_held_message; // inbuxa: mail held for review
|
||||
pub mod inbuxa_hold_export; // inbuxa: legal hold exports
|
||||
pub mod inbuxa_explanation; // inbuxa: "Explain this" with the local model
|
||||
pub mod inbuxa_protocol_policy; // inbuxa: legacy protocols off
|
||||
|
||||
@@ -64,6 +64,9 @@ impl Response<'_> {
|
||||
GetResponseMethod::LogSettings(response) => {
|
||||
response.eval_jptr(path, &mut results)
|
||||
}
|
||||
GetResponseMethod::DlpSettings(response) => {
|
||||
response.eval_jptr(path, &mut results)
|
||||
}
|
||||
GetResponseMethod::DataInventory(response) => {
|
||||
response.eval_jptr(path, &mut results)
|
||||
}
|
||||
@@ -82,6 +85,18 @@ impl Response<'_> {
|
||||
GetResponseMethod::LegalHold(response) => {
|
||||
response.eval_jptr(path, &mut results)
|
||||
}
|
||||
GetResponseMethod::MailRule(response) => {
|
||||
response.eval_jptr(path, &mut results)
|
||||
}
|
||||
GetResponseMethod::SecurityAcceptance(response) => {
|
||||
response.eval_jptr(path, &mut results)
|
||||
}
|
||||
GetResponseMethod::Journal(response) => {
|
||||
response.eval_jptr(path, &mut results)
|
||||
}
|
||||
GetResponseMethod::HeldMessage(response) => {
|
||||
response.eval_jptr(path, &mut results)
|
||||
}
|
||||
GetResponseMethod::HoldExport(response) => {
|
||||
response.eval_jptr(path, &mut results)
|
||||
}
|
||||
|
||||
@@ -47,12 +47,17 @@ impl Response<'_> {
|
||||
GetRequestMethod::DeletedAccount(request) => request.resolve_references(self)?,
|
||||
GetRequestMethod::AiLimits(request) => request.resolve_references(self)?,
|
||||
GetRequestMethod::LogSettings(request) => request.resolve_references(self)?,
|
||||
GetRequestMethod::DlpSettings(request) => request.resolve_references(self)?,
|
||||
GetRequestMethod::DataInventory(request) => request.resolve_references(self)?,
|
||||
GetRequestMethod::InventorySnapshot(request) => request.resolve_references(self)?,
|
||||
GetRequestMethod::AuditEvent(request) => request.resolve_references(self)?,
|
||||
GetRequestMethod::AuditSettings(request) => request.resolve_references(self)?,
|
||||
GetRequestMethod::AccountLock(request) => request.resolve_references(self)?,
|
||||
GetRequestMethod::LegalHold(request) => request.resolve_references(self)?,
|
||||
GetRequestMethod::MailRule(request) => request.resolve_references(self)?,
|
||||
GetRequestMethod::SecurityAcceptance(request) => request.resolve_references(self)?,
|
||||
GetRequestMethod::Journal(request) => request.resolve_references(self)?,
|
||||
GetRequestMethod::HeldMessage(request) => request.resolve_references(self)?,
|
||||
GetRequestMethod::HoldExport(request) => request.resolve_references(self)?,
|
||||
GetRequestMethod::ProtocolPolicy(request) => request.resolve_references(self)?,
|
||||
GetRequestMethod::TenantProtocolPolicy(request) => {
|
||||
@@ -104,6 +109,9 @@ impl Response<'_> {
|
||||
SetRequestMethod::LogSettings(request) => {
|
||||
request.resolve_references(self, 1, false)?
|
||||
}
|
||||
SetRequestMethod::DlpSettings(request) => {
|
||||
request.resolve_references(self, 1, false)?
|
||||
}
|
||||
SetRequestMethod::Explanation(request) => {
|
||||
request.resolve_references(self, 1, false)?
|
||||
}
|
||||
@@ -122,6 +130,18 @@ impl Response<'_> {
|
||||
SetRequestMethod::LegalHold(request) => {
|
||||
request.resolve_references(self, 1, false)?
|
||||
}
|
||||
SetRequestMethod::MailRule(request) => {
|
||||
request.resolve_references(self, 1, false)?
|
||||
}
|
||||
SetRequestMethod::SecurityAcceptance(request) => {
|
||||
request.resolve_references(self, 1, false)?
|
||||
}
|
||||
SetRequestMethod::Journal(request) => {
|
||||
request.resolve_references(self, 1, false)?
|
||||
}
|
||||
SetRequestMethod::HeldMessage(request) => {
|
||||
request.resolve_references(self, 1, false)?
|
||||
}
|
||||
SetRequestMethod::HoldExport(request) => {
|
||||
request.resolve_references(self, 1, false)?
|
||||
}
|
||||
|
||||
@@ -50,6 +50,7 @@ pub enum MethodObject {
|
||||
// inbuxa: AI call limits
|
||||
AiLimits,
|
||||
LogSettings,
|
||||
DlpSettings,
|
||||
DataInventory,
|
||||
InventorySnapshot,
|
||||
// inbuxa: "Explain this" with the local model
|
||||
@@ -65,6 +66,13 @@ pub enum MethodObject {
|
||||
LegalHold,
|
||||
HoldExport,
|
||||
ProtocolPolicy,
|
||||
// inbuxa: DLP and mail flow rules
|
||||
MailRule,
|
||||
// inbuxa: accepted security to-do items
|
||||
SecurityAcceptance,
|
||||
HeldMessage,
|
||||
// inbuxa: journaling
|
||||
Journal,
|
||||
TenantProtocolPolicy,
|
||||
}
|
||||
|
||||
@@ -93,6 +101,7 @@ impl MethodObject {
|
||||
MethodObject::DeletedAccount => Capability::Inbuxa,
|
||||
MethodObject::AiLimits => Capability::Inbuxa,
|
||||
MethodObject::LogSettings => Capability::Inbuxa,
|
||||
MethodObject::DlpSettings => Capability::Inbuxa,
|
||||
MethodObject::DataInventory => Capability::Inbuxa,
|
||||
MethodObject::InventorySnapshot => Capability::Inbuxa,
|
||||
MethodObject::Explanation => Capability::Inbuxa,
|
||||
@@ -102,7 +111,11 @@ impl MethodObject {
|
||||
| MethodObject::AuditVerification
|
||||
| MethodObject::AccountLock
|
||||
| MethodObject::LegalHold
|
||||
| MethodObject::HoldExport => Capability::Inbuxa,
|
||||
| MethodObject::HoldExport
|
||||
| MethodObject::MailRule
|
||||
| MethodObject::SecurityAcceptance
|
||||
| MethodObject::HeldMessage
|
||||
| MethodObject::Journal => Capability::Inbuxa,
|
||||
MethodObject::ProtocolPolicy => Capability::Inbuxa,
|
||||
MethodObject::TenantProtocolPolicy => Capability::Inbuxa,
|
||||
}
|
||||
@@ -283,9 +296,11 @@ impl MethodName {
|
||||
(MethodFunction::Get, MethodObject::AiLimits) => "inbuxa:AiLimits/get",
|
||||
(MethodFunction::Set, MethodObject::AiLimits) => "inbuxa:AiLimits/set",
|
||||
(MethodFunction::Get, MethodObject::LogSettings) => "inbuxa:LogSettings/get",
|
||||
(MethodFunction::Get, MethodObject::DlpSettings) => "inbuxa:DlpSettings/get",
|
||||
(MethodFunction::Get, MethodObject::DataInventory) => "inbuxa:DataInventory/get",
|
||||
(MethodFunction::Get, MethodObject::InventorySnapshot) => "inbuxa:InventorySnapshot/get",
|
||||
(MethodFunction::Set, MethodObject::LogSettings) => "inbuxa:LogSettings/set",
|
||||
(MethodFunction::Set, MethodObject::DlpSettings) => "inbuxa:DlpSettings/set",
|
||||
(MethodFunction::Set, MethodObject::Explanation) => "inbuxa:Explanation/set",
|
||||
(MethodFunction::Get, MethodObject::AuditEvent) => "inbuxa:AuditEvent/get",
|
||||
(MethodFunction::Query, MethodObject::AuditEvent) => "inbuxa:AuditEvent/query",
|
||||
@@ -296,6 +311,14 @@ impl MethodName {
|
||||
(MethodFunction::Set, MethodObject::AccountLock) => "inbuxa:AccountLock/set",
|
||||
(MethodFunction::Get, MethodObject::LegalHold) => "inbuxa:LegalHold/get",
|
||||
(MethodFunction::Set, MethodObject::LegalHold) => "inbuxa:LegalHold/set",
|
||||
(MethodFunction::Get, MethodObject::MailRule) => "inbuxa:MailRule/get",
|
||||
(MethodFunction::Set, MethodObject::MailRule) => "inbuxa:MailRule/set",
|
||||
(MethodFunction::Get, MethodObject::SecurityAcceptance) => "inbuxa:SecurityAcceptance/get",
|
||||
(MethodFunction::Set, MethodObject::SecurityAcceptance) => "inbuxa:SecurityAcceptance/set",
|
||||
(MethodFunction::Get, MethodObject::Journal) => "inbuxa:Journal/get",
|
||||
(MethodFunction::Set, MethodObject::Journal) => "inbuxa:Journal/set",
|
||||
(MethodFunction::Get, MethodObject::HeldMessage) => "inbuxa:HeldMessage/get",
|
||||
(MethodFunction::Set, MethodObject::HeldMessage) => "inbuxa:HeldMessage/set",
|
||||
(MethodFunction::Get, MethodObject::HoldExport) => "inbuxa:HoldExport/get",
|
||||
(MethodFunction::Set, MethodObject::HoldExport) => "inbuxa:HoldExport/set",
|
||||
(MethodFunction::Set, MethodObject::AuditVerification) => {
|
||||
@@ -435,9 +458,11 @@ impl MethodName {
|
||||
"inbuxa:AiLimits/get" => (MethodObject::AiLimits, MethodFunction::Get),
|
||||
"inbuxa:AiLimits/set" => (MethodObject::AiLimits, MethodFunction::Set),
|
||||
"inbuxa:LogSettings/get" => (MethodObject::LogSettings, MethodFunction::Get),
|
||||
"inbuxa:DlpSettings/get" => (MethodObject::DlpSettings, MethodFunction::Get),
|
||||
"inbuxa:DataInventory/get" => (MethodObject::DataInventory, MethodFunction::Get),
|
||||
"inbuxa:InventorySnapshot/get" => (MethodObject::InventorySnapshot, MethodFunction::Get),
|
||||
"inbuxa:LogSettings/set" => (MethodObject::LogSettings, MethodFunction::Set),
|
||||
"inbuxa:DlpSettings/set" => (MethodObject::DlpSettings, MethodFunction::Set),
|
||||
"inbuxa:Explanation/set" => (MethodObject::Explanation, MethodFunction::Set),
|
||||
"inbuxa:AuditEvent/get" => (MethodObject::AuditEvent, MethodFunction::Get),
|
||||
"inbuxa:AuditEvent/query" => (MethodObject::AuditEvent, MethodFunction::Query),
|
||||
@@ -448,6 +473,14 @@ impl MethodName {
|
||||
"inbuxa:AccountLock/set" => (MethodObject::AccountLock, MethodFunction::Set),
|
||||
"inbuxa:LegalHold/get" => (MethodObject::LegalHold, MethodFunction::Get),
|
||||
"inbuxa:LegalHold/set" => (MethodObject::LegalHold, MethodFunction::Set),
|
||||
"inbuxa:MailRule/get" => (MethodObject::MailRule, MethodFunction::Get),
|
||||
"inbuxa:MailRule/set" => (MethodObject::MailRule, MethodFunction::Set),
|
||||
"inbuxa:SecurityAcceptance/get" => (MethodObject::SecurityAcceptance, MethodFunction::Get),
|
||||
"inbuxa:SecurityAcceptance/set" => (MethodObject::SecurityAcceptance, MethodFunction::Set),
|
||||
"inbuxa:Journal/get" => (MethodObject::Journal, MethodFunction::Get),
|
||||
"inbuxa:Journal/set" => (MethodObject::Journal, MethodFunction::Set),
|
||||
"inbuxa:HeldMessage/get" => (MethodObject::HeldMessage, MethodFunction::Get),
|
||||
"inbuxa:HeldMessage/set" => (MethodObject::HeldMessage, MethodFunction::Set),
|
||||
"inbuxa:HoldExport/get" => (MethodObject::HoldExport, MethodFunction::Get),
|
||||
"inbuxa:HoldExport/set" => (MethodObject::HoldExport, MethodFunction::Set),
|
||||
"inbuxa:AuditVerification/set" => (MethodObject::AuditVerification, MethodFunction::Set),
|
||||
@@ -509,6 +542,7 @@ impl Display for MethodObject {
|
||||
MethodObject::DeletedAccount => "inbuxa:DeletedAccount",
|
||||
MethodObject::AiLimits => "inbuxa:AiLimits",
|
||||
MethodObject::LogSettings => "inbuxa:LogSettings",
|
||||
MethodObject::DlpSettings => "inbuxa:DlpSettings",
|
||||
MethodObject::DataInventory => "inbuxa:DataInventory",
|
||||
MethodObject::InventorySnapshot => "inbuxa:InventorySnapshot",
|
||||
MethodObject::Explanation => "inbuxa:Explanation",
|
||||
@@ -518,6 +552,10 @@ impl Display for MethodObject {
|
||||
MethodObject::AuditVerification => "inbuxa:AuditVerification",
|
||||
MethodObject::AccountLock => "inbuxa:AccountLock",
|
||||
MethodObject::LegalHold => "inbuxa:LegalHold",
|
||||
MethodObject::MailRule => "inbuxa:MailRule",
|
||||
MethodObject::SecurityAcceptance => "inbuxa:SecurityAcceptance",
|
||||
MethodObject::Journal => "inbuxa:Journal",
|
||||
MethodObject::HeldMessage => "inbuxa:HeldMessage",
|
||||
MethodObject::HoldExport => "inbuxa:HoldExport",
|
||||
MethodObject::ProtocolPolicy => "inbuxa:ProtocolPolicy",
|
||||
MethodObject::TenantProtocolPolicy => "inbuxa:TenantProtocolPolicy",
|
||||
|
||||
@@ -117,12 +117,17 @@ pub enum GetRequestMethod {
|
||||
DeletedAccount(Box<GetRequest<crate::object::inbuxa_deleted_account::DeletedAccount>>),
|
||||
AiLimits(Box<GetRequest<crate::object::inbuxa_ai_limits::AiLimits>>),
|
||||
LogSettings(Box<GetRequest<crate::object::inbuxa_log_settings::LogSettings>>),
|
||||
DlpSettings(Box<GetRequest<crate::object::inbuxa_dlp_settings::DlpSettings>>),
|
||||
DataInventory(Box<GetRequest<crate::object::inbuxa_data_inventory::DataInventory>>),
|
||||
InventorySnapshot(Box<GetRequest<crate::object::inbuxa_inventory_snapshot::InventorySnapshot>>),
|
||||
AuditEvent(Box<GetRequest<crate::object::inbuxa_audit::AuditEvent>>),
|
||||
AuditSettings(Box<GetRequest<crate::object::inbuxa_audit::AuditSettings>>),
|
||||
AccountLock(Box<GetRequest<crate::object::inbuxa_account_lock::AccountLock>>),
|
||||
LegalHold(Box<GetRequest<crate::object::inbuxa_legal_hold::LegalHold>>),
|
||||
MailRule(Box<GetRequest<crate::object::inbuxa_mail_rule::MailRule>>),
|
||||
SecurityAcceptance(Box<GetRequest<crate::object::inbuxa_security_acceptance::SecurityAcceptance>>),
|
||||
Journal(Box<GetRequest<crate::object::inbuxa_journal::Journal>>),
|
||||
HeldMessage(Box<GetRequest<crate::object::inbuxa_held_message::HeldMessage>>),
|
||||
HoldExport(Box<GetRequest<crate::object::inbuxa_hold_export::HoldExport>>),
|
||||
ProtocolPolicy(Box<GetRequest<crate::object::inbuxa_protocol_policy::ProtocolPolicy>>),
|
||||
TenantProtocolPolicy(
|
||||
@@ -152,12 +157,19 @@ pub enum SetRequestMethod<'x> {
|
||||
DeletedAccount(Box<SetRequest<'x, crate::object::inbuxa_deleted_account::DeletedAccount>>),
|
||||
AiLimits(Box<SetRequest<'x, crate::object::inbuxa_ai_limits::AiLimits>>),
|
||||
LogSettings(Box<SetRequest<'x, crate::object::inbuxa_log_settings::LogSettings>>),
|
||||
DlpSettings(Box<SetRequest<'x, crate::object::inbuxa_dlp_settings::DlpSettings>>),
|
||||
Explanation(Box<SetRequest<'x, crate::object::inbuxa_explanation::Explanation>>),
|
||||
AuditSettings(Box<SetRequest<'x, crate::object::inbuxa_audit::AuditSettings>>),
|
||||
AuditExport(Box<SetRequest<'x, crate::object::inbuxa_audit::AuditExport>>),
|
||||
AuditVerification(Box<SetRequest<'x, crate::object::inbuxa_audit::AuditVerification>>),
|
||||
AccountLock(Box<SetRequest<'x, crate::object::inbuxa_account_lock::AccountLock>>),
|
||||
LegalHold(Box<SetRequest<'x, crate::object::inbuxa_legal_hold::LegalHold>>),
|
||||
MailRule(Box<SetRequest<'x, crate::object::inbuxa_mail_rule::MailRule>>),
|
||||
SecurityAcceptance(
|
||||
Box<SetRequest<'x, crate::object::inbuxa_security_acceptance::SecurityAcceptance>>,
|
||||
),
|
||||
Journal(Box<SetRequest<'x, crate::object::inbuxa_journal::Journal>>),
|
||||
HeldMessage(Box<SetRequest<'x, crate::object::inbuxa_held_message::HeldMessage>>),
|
||||
HoldExport(Box<SetRequest<'x, crate::object::inbuxa_hold_export::HoldExport>>),
|
||||
ProtocolPolicy(Box<SetRequest<'x, crate::object::inbuxa_protocol_policy::ProtocolPolicy>>),
|
||||
TenantProtocolPolicy(
|
||||
|
||||
@@ -176,6 +176,13 @@ impl<'de> Visitor<'de> for CallVisitor {
|
||||
return Err(de::Error::invalid_length(1, &self));
|
||||
}
|
||||
},
|
||||
(MethodFunction::Get, MethodObject::DlpSettings) => match seq.next_element() {
|
||||
Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::DlpSettings(value)),
|
||||
Err(err) => RequestMethod::invalid(err),
|
||||
Ok(None) => {
|
||||
return Err(de::Error::invalid_length(1, &self));
|
||||
}
|
||||
},
|
||||
(MethodFunction::Get, MethodObject::DataInventory) => match seq.next_element() {
|
||||
Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::DataInventory(value)),
|
||||
Err(err) => RequestMethod::invalid(err),
|
||||
@@ -378,6 +385,13 @@ impl<'de> Visitor<'de> for CallVisitor {
|
||||
return Err(de::Error::invalid_length(1, &self));
|
||||
}
|
||||
},
|
||||
(MethodFunction::Set, MethodObject::DlpSettings) => match seq.next_element() {
|
||||
Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::DlpSettings(value)),
|
||||
Err(err) => RequestMethod::invalid(err),
|
||||
Ok(None) => {
|
||||
return Err(de::Error::invalid_length(1, &self));
|
||||
}
|
||||
},
|
||||
(MethodFunction::Set, MethodObject::Explanation) => match seq.next_element() {
|
||||
Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::Explanation(value)),
|
||||
Err(err) => RequestMethod::invalid(err),
|
||||
@@ -609,6 +623,66 @@ impl<'de> Visitor<'de> for CallVisitor {
|
||||
return Err(de::Error::invalid_length(1, &self));
|
||||
}
|
||||
},
|
||||
// inbuxa: mail held for review
|
||||
(MethodFunction::Get, MethodObject::HeldMessage) => match seq.next_element() {
|
||||
Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::HeldMessage(value)),
|
||||
Err(err) => RequestMethod::invalid(err),
|
||||
Ok(None) => {
|
||||
return Err(de::Error::invalid_length(1, &self));
|
||||
}
|
||||
},
|
||||
(MethodFunction::Set, MethodObject::HeldMessage) => match seq.next_element() {
|
||||
Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::HeldMessage(value)),
|
||||
Err(err) => RequestMethod::invalid(err),
|
||||
Ok(None) => {
|
||||
return Err(de::Error::invalid_length(1, &self));
|
||||
}
|
||||
},
|
||||
// inbuxa: DLP and mail flow rules
|
||||
(MethodFunction::Get, MethodObject::MailRule) => match seq.next_element() {
|
||||
Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::MailRule(value)),
|
||||
Err(err) => RequestMethod::invalid(err),
|
||||
Ok(None) => {
|
||||
return Err(de::Error::invalid_length(1, &self));
|
||||
}
|
||||
},
|
||||
(MethodFunction::Set, MethodObject::MailRule) => match seq.next_element() {
|
||||
Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::MailRule(value)),
|
||||
Err(err) => RequestMethod::invalid(err),
|
||||
Ok(None) => {
|
||||
return Err(de::Error::invalid_length(1, &self));
|
||||
}
|
||||
},
|
||||
// inbuxa: accepted security to-do items
|
||||
(MethodFunction::Get, MethodObject::SecurityAcceptance) => match seq.next_element() {
|
||||
Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::SecurityAcceptance(value)),
|
||||
Err(err) => RequestMethod::invalid(err),
|
||||
Ok(None) => {
|
||||
return Err(de::Error::invalid_length(1, &self));
|
||||
}
|
||||
},
|
||||
(MethodFunction::Set, MethodObject::SecurityAcceptance) => match seq.next_element() {
|
||||
Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::SecurityAcceptance(value)),
|
||||
Err(err) => RequestMethod::invalid(err),
|
||||
Ok(None) => {
|
||||
return Err(de::Error::invalid_length(1, &self));
|
||||
}
|
||||
},
|
||||
// inbuxa: journaling
|
||||
(MethodFunction::Get, MethodObject::Journal) => match seq.next_element() {
|
||||
Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::Journal(value)),
|
||||
Err(err) => RequestMethod::invalid(err),
|
||||
Ok(None) => {
|
||||
return Err(de::Error::invalid_length(1, &self));
|
||||
}
|
||||
},
|
||||
(MethodFunction::Set, MethodObject::Journal) => match seq.next_element() {
|
||||
Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::Journal(value)),
|
||||
Err(err) => RequestMethod::invalid(err),
|
||||
Ok(None) => {
|
||||
return Err(de::Error::invalid_length(1, &self));
|
||||
}
|
||||
},
|
||||
// inbuxa: legal hold
|
||||
(MethodFunction::Get, MethodObject::LegalHold) => match seq.next_element() {
|
||||
Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::LegalHold(value)),
|
||||
|
||||
@@ -104,12 +104,17 @@ pub enum GetResponseMethod {
|
||||
DeletedAccount(GetResponse<crate::object::inbuxa_deleted_account::DeletedAccount>),
|
||||
AiLimits(GetResponse<crate::object::inbuxa_ai_limits::AiLimits>),
|
||||
LogSettings(GetResponse<crate::object::inbuxa_log_settings::LogSettings>),
|
||||
DlpSettings(GetResponse<crate::object::inbuxa_dlp_settings::DlpSettings>),
|
||||
DataInventory(GetResponse<crate::object::inbuxa_data_inventory::DataInventory>),
|
||||
InventorySnapshot(GetResponse<crate::object::inbuxa_inventory_snapshot::InventorySnapshot>),
|
||||
AuditEvent(GetResponse<crate::object::inbuxa_audit::AuditEvent>),
|
||||
AuditSettings(GetResponse<crate::object::inbuxa_audit::AuditSettings>),
|
||||
AccountLock(GetResponse<crate::object::inbuxa_account_lock::AccountLock>),
|
||||
LegalHold(GetResponse<crate::object::inbuxa_legal_hold::LegalHold>),
|
||||
MailRule(GetResponse<crate::object::inbuxa_mail_rule::MailRule>),
|
||||
SecurityAcceptance(GetResponse<crate::object::inbuxa_security_acceptance::SecurityAcceptance>),
|
||||
Journal(GetResponse<crate::object::inbuxa_journal::Journal>),
|
||||
HeldMessage(GetResponse<crate::object::inbuxa_held_message::HeldMessage>),
|
||||
HoldExport(GetResponse<crate::object::inbuxa_hold_export::HoldExport>),
|
||||
ProtocolPolicy(GetResponse<crate::object::inbuxa_protocol_policy::ProtocolPolicy>),
|
||||
TenantProtocolPolicy(
|
||||
@@ -140,11 +145,18 @@ pub enum SetResponseMethod {
|
||||
DeletedAccount(Box<SetResponse<crate::object::inbuxa_deleted_account::DeletedAccount>>),
|
||||
AiLimits(Box<SetResponse<crate::object::inbuxa_ai_limits::AiLimits>>),
|
||||
LogSettings(Box<SetResponse<crate::object::inbuxa_log_settings::LogSettings>>),
|
||||
DlpSettings(Box<SetResponse<crate::object::inbuxa_dlp_settings::DlpSettings>>),
|
||||
AuditSettings(Box<SetResponse<crate::object::inbuxa_audit::AuditSettings>>),
|
||||
AuditExport(Box<SetResponse<crate::object::inbuxa_audit::AuditExport>>),
|
||||
AuditVerification(Box<SetResponse<crate::object::inbuxa_audit::AuditVerification>>),
|
||||
AccountLock(Box<SetResponse<crate::object::inbuxa_account_lock::AccountLock>>),
|
||||
LegalHold(Box<SetResponse<crate::object::inbuxa_legal_hold::LegalHold>>),
|
||||
MailRule(Box<SetResponse<crate::object::inbuxa_mail_rule::MailRule>>),
|
||||
SecurityAcceptance(
|
||||
Box<SetResponse<crate::object::inbuxa_security_acceptance::SecurityAcceptance>>,
|
||||
),
|
||||
Journal(Box<SetResponse<crate::object::inbuxa_journal::Journal>>),
|
||||
HeldMessage(Box<SetResponse<crate::object::inbuxa_held_message::HeldMessage>>),
|
||||
HoldExport(Box<SetResponse<crate::object::inbuxa_hold_export::HoldExport>>),
|
||||
Explanation(Box<SetResponse<crate::object::inbuxa_explanation::Explanation>>),
|
||||
ProtocolPolicy(Box<SetResponse<crate::object::inbuxa_protocol_policy::ProtocolPolicy>>),
|
||||
@@ -359,6 +371,12 @@ impl<'x> From<GetResponse<crate::object::inbuxa_log_settings::LogSettings>> for
|
||||
}
|
||||
}
|
||||
|
||||
impl<'x> From<GetResponse<crate::object::inbuxa_dlp_settings::DlpSettings>> for ResponseMethod<'x> {
|
||||
fn from(value: GetResponse<crate::object::inbuxa_dlp_settings::DlpSettings>) -> Self {
|
||||
ResponseMethod::Get(GetResponseMethod::DlpSettings(value))
|
||||
}
|
||||
}
|
||||
|
||||
impl<'x> From<GetResponse<crate::object::inbuxa_data_inventory::DataInventory>> for ResponseMethod<'x> {
|
||||
fn from(value: GetResponse<crate::object::inbuxa_data_inventory::DataInventory>) -> Self {
|
||||
ResponseMethod::Get(GetResponseMethod::DataInventory(value))
|
||||
@@ -383,6 +401,12 @@ impl<'x> From<SetResponse<crate::object::inbuxa_log_settings::LogSettings>> for
|
||||
}
|
||||
}
|
||||
|
||||
impl<'x> From<SetResponse<crate::object::inbuxa_dlp_settings::DlpSettings>> for ResponseMethod<'x> {
|
||||
fn from(value: SetResponse<crate::object::inbuxa_dlp_settings::DlpSettings>) -> Self {
|
||||
ResponseMethod::Set(SetResponseMethod::DlpSettings(Box::new(value)))
|
||||
}
|
||||
}
|
||||
|
||||
impl<'x> From<SetResponse<crate::object::inbuxa_explanation::Explanation>> for ResponseMethod<'x> {
|
||||
fn from(value: SetResponse<crate::object::inbuxa_explanation::Explanation>) -> Self {
|
||||
ResponseMethod::Set(SetResponseMethod::Explanation(Box::new(value)))
|
||||
@@ -799,6 +823,60 @@ impl<'x> From<SetResponse<crate::object::inbuxa_account_lock::AccountLock>> for
|
||||
}
|
||||
|
||||
// inbuxa: legal hold
|
||||
impl<'x> From<GetResponse<crate::object::inbuxa_held_message::HeldMessage>> for ResponseMethod<'x> {
|
||||
fn from(value: GetResponse<crate::object::inbuxa_held_message::HeldMessage>) -> Self {
|
||||
ResponseMethod::Get(GetResponseMethod::HeldMessage(value))
|
||||
}
|
||||
}
|
||||
|
||||
impl<'x> From<SetResponse<crate::object::inbuxa_held_message::HeldMessage>> for ResponseMethod<'x> {
|
||||
fn from(value: SetResponse<crate::object::inbuxa_held_message::HeldMessage>) -> Self {
|
||||
ResponseMethod::Set(SetResponseMethod::HeldMessage(Box::new(value)))
|
||||
}
|
||||
}
|
||||
|
||||
// inbuxa: accepted security to-do items
|
||||
impl<'x> From<GetResponse<crate::object::inbuxa_security_acceptance::SecurityAcceptance>>
|
||||
for ResponseMethod<'x>
|
||||
{
|
||||
fn from(value: GetResponse<crate::object::inbuxa_security_acceptance::SecurityAcceptance>) -> Self {
|
||||
ResponseMethod::Get(GetResponseMethod::SecurityAcceptance(value))
|
||||
}
|
||||
}
|
||||
|
||||
impl<'x> From<SetResponse<crate::object::inbuxa_security_acceptance::SecurityAcceptance>>
|
||||
for ResponseMethod<'x>
|
||||
{
|
||||
fn from(value: SetResponse<crate::object::inbuxa_security_acceptance::SecurityAcceptance>) -> Self {
|
||||
ResponseMethod::Set(SetResponseMethod::SecurityAcceptance(Box::new(value)))
|
||||
}
|
||||
}
|
||||
|
||||
impl<'x> From<GetResponse<crate::object::inbuxa_mail_rule::MailRule>> for ResponseMethod<'x> {
|
||||
fn from(value: GetResponse<crate::object::inbuxa_mail_rule::MailRule>) -> Self {
|
||||
ResponseMethod::Get(GetResponseMethod::MailRule(value))
|
||||
}
|
||||
}
|
||||
|
||||
impl<'x> From<SetResponse<crate::object::inbuxa_mail_rule::MailRule>> for ResponseMethod<'x> {
|
||||
fn from(value: SetResponse<crate::object::inbuxa_mail_rule::MailRule>) -> Self {
|
||||
ResponseMethod::Set(SetResponseMethod::MailRule(Box::new(value)))
|
||||
}
|
||||
}
|
||||
|
||||
// inbuxa: journaling
|
||||
impl<'x> From<GetResponse<crate::object::inbuxa_journal::Journal>> for ResponseMethod<'x> {
|
||||
fn from(value: GetResponse<crate::object::inbuxa_journal::Journal>) -> Self {
|
||||
ResponseMethod::Get(GetResponseMethod::Journal(value))
|
||||
}
|
||||
}
|
||||
|
||||
impl<'x> From<SetResponse<crate::object::inbuxa_journal::Journal>> for ResponseMethod<'x> {
|
||||
fn from(value: SetResponse<crate::object::inbuxa_journal::Journal>) -> Self {
|
||||
ResponseMethod::Set(SetResponseMethod::Journal(Box::new(value)))
|
||||
}
|
||||
}
|
||||
|
||||
impl<'x> From<GetResponse<crate::object::inbuxa_legal_hold::LegalHold>> for ResponseMethod<'x> {
|
||||
fn from(value: GetResponse<crate::object::inbuxa_legal_hold::LegalHold>) -> Self {
|
||||
ResponseMethod::Get(GetResponseMethod::LegalHold(value))
|
||||
|
||||
@@ -92,6 +92,7 @@ impl JmapAuthorization for AccessToken {
|
||||
GetRequestMethod::AiLimits(_) => Permission::SysSpamLlmGet,
|
||||
// inbuxa: log file retention, with the tracers' permissions
|
||||
GetRequestMethod::LogSettings(_) => Permission::SysTracerGet,
|
||||
GetRequestMethod::DlpSettings(_) => Permission::SysDlpPolicyGet,
|
||||
// inbuxa: personal-data catalog, the inventory and its history
|
||||
GetRequestMethod::DataInventory(_) | GetRequestMethod::InventorySnapshot(_) => {
|
||||
Permission::SysComplianceGet
|
||||
@@ -103,7 +104,24 @@ impl JmapAuthorization for AccessToken {
|
||||
// inbuxa: account lock (AL-12)
|
||||
GetRequestMethod::AccountLock(_) => Permission::SysAccountLockGet,
|
||||
GetRequestMethod::LegalHold(_) => Permission::SysLegalHoldGet,
|
||||
// inbuxa: DLP and mail flow rules share an object; either
|
||||
// permission reaches it, and the handler shows each kind
|
||||
// only to those who may see it
|
||||
// inbuxa: mail held for review (§2.8)
|
||||
GetRequestMethod::HeldMessage(_) => Permission::SysDlpReviewGet,
|
||||
GetRequestMethod::MailRule(_) => {
|
||||
if self.has_permission(Permission::SysMailRuleGet) {
|
||||
Permission::SysMailRuleGet
|
||||
} else {
|
||||
Permission::SysDlpPolicyGet
|
||||
}
|
||||
}
|
||||
// inbuxa: journaling (JR-18)
|
||||
GetRequestMethod::Journal(_) => Permission::SysJournalGet,
|
||||
GetRequestMethod::HoldExport(_) => Permission::SysLegalHoldExport,
|
||||
// inbuxa: accepted security items are read by whoever may
|
||||
// see the server's security settings
|
||||
GetRequestMethod::SecurityAcceptance(_) => Permission::SysSecurityGet,
|
||||
// inbuxa: legacy protocols off. It takes listeners away and
|
||||
// puts them back, so it takes the listener's permissions
|
||||
GetRequestMethod::ProtocolPolicy(_) => Permission::SysNetworkListenerGet,
|
||||
@@ -215,6 +233,13 @@ impl JmapAuthorization for AccessToken {
|
||||
Permission::SysTracerUpdate,
|
||||
Permission::SysTracerUpdate,
|
||||
),
|
||||
SetRequestMethod::DlpSettings(s) => validate_set(
|
||||
s,
|
||||
self,
|
||||
Permission::SysDlpPolicyUpdate,
|
||||
Permission::SysDlpPolicyUpdate,
|
||||
Permission::SysDlpPolicyUpdate,
|
||||
),
|
||||
// inbuxa: the audit log (AU-7, AU-9, AU-11)
|
||||
SetRequestMethod::AuditSettings(s) => validate_set(
|
||||
s,
|
||||
@@ -247,6 +272,50 @@ impl JmapAuthorization for AccessToken {
|
||||
Permission::SysLegalHoldUpdate,
|
||||
Permission::SysLegalHoldUpdate,
|
||||
),
|
||||
// inbuxa: releasing or rejecting held mail (§2.8)
|
||||
SetRequestMethod::HeldMessage(s) => validate_set(
|
||||
s,
|
||||
self,
|
||||
Permission::SysDlpReviewUpdate,
|
||||
Permission::SysDlpReviewUpdate,
|
||||
Permission::SysDlpReviewUpdate,
|
||||
),
|
||||
// inbuxa: DLP and mail flow rules: either change
|
||||
// permission gets in; the handler checks each rule's kind
|
||||
SetRequestMethod::MailRule(_) => {
|
||||
if self.has_permission(Permission::SysMailRuleUpdate)
|
||||
|| self.has_permission(Permission::SysDlpPolicyUpdate)
|
||||
{
|
||||
Ok(())
|
||||
} else {
|
||||
Err(trc::JmapEvent::Forbidden
|
||||
.into_err()
|
||||
.details("You are not authorized to change mail rules"))
|
||||
}
|
||||
}
|
||||
// inbuxa: journaling (JR-18)
|
||||
SetRequestMethod::Journal(s) => validate_set(
|
||||
s,
|
||||
self,
|
||||
Permission::SysJournalUpdate,
|
||||
Permission::SysJournalUpdate,
|
||||
Permission::SysJournalUpdate,
|
||||
),
|
||||
// inbuxa: accepting a security to-do item, or removing
|
||||
// an acceptance; nothing is ever edited
|
||||
SetRequestMethod::SecurityAcceptance(s) => {
|
||||
if s.update.as_ref().is_some_and(|u| !u.is_empty()) {
|
||||
Err(trc::JmapEvent::Forbidden
|
||||
.into_err()
|
||||
.details("An acceptance is replaced, not edited"))
|
||||
} else if self.has_permission(Permission::SysSecurityAccept) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(trc::JmapEvent::Forbidden
|
||||
.into_err()
|
||||
.details("You are not authorized to accept security items"))
|
||||
}
|
||||
}
|
||||
// inbuxa: LH-12, exporting held data
|
||||
SetRequestMethod::HoldExport(s) => validate_set(
|
||||
s,
|
||||
@@ -397,6 +466,7 @@ impl JmapAuthorization for AccessToken {
|
||||
| MethodObject::DeletedAccount
|
||||
| MethodObject::AiLimits
|
||||
| MethodObject::LogSettings
|
||||
| MethodObject::DlpSettings
|
||||
| MethodObject::DataInventory
|
||||
| MethodObject::InventorySnapshot
|
||||
| MethodObject::Explanation
|
||||
@@ -407,6 +477,10 @@ impl JmapAuthorization for AccessToken {
|
||||
| MethodObject::AccountLock
|
||||
| MethodObject::LegalHold
|
||||
| MethodObject::HoldExport
|
||||
| MethodObject::MailRule
|
||||
| MethodObject::SecurityAcceptance
|
||||
| MethodObject::HeldMessage
|
||||
| MethodObject::Journal
|
||||
| MethodObject::ProtocolPolicy
|
||||
| MethodObject::TenantProtocolPolicy => Permission::JmapEmailChanges,
|
||||
// inbuxa: x:MaskedEmail/changes reads what /get reads
|
||||
|
||||
@@ -264,6 +264,9 @@ impl RequestHandler for Server {
|
||||
SetResponseMethod::LogSettings(set_response) => {
|
||||
set_response.update_created_ids(&mut response);
|
||||
}
|
||||
SetResponseMethod::DlpSettings(set_response) => {
|
||||
set_response.update_created_ids(&mut response);
|
||||
}
|
||||
SetResponseMethod::AuditSettings(set_response) => {
|
||||
set_response.update_created_ids(&mut response);
|
||||
}
|
||||
@@ -279,6 +282,18 @@ impl RequestHandler for Server {
|
||||
SetResponseMethod::LegalHold(set_response) => {
|
||||
set_response.update_created_ids(&mut response);
|
||||
}
|
||||
SetResponseMethod::MailRule(set_response) => {
|
||||
set_response.update_created_ids(&mut response);
|
||||
}
|
||||
SetResponseMethod::SecurityAcceptance(set_response) => {
|
||||
set_response.update_created_ids(&mut response);
|
||||
}
|
||||
SetResponseMethod::Journal(set_response) => {
|
||||
set_response.update_created_ids(&mut response);
|
||||
}
|
||||
SetResponseMethod::HeldMessage(set_response) => {
|
||||
set_response.update_created_ids(&mut response);
|
||||
}
|
||||
SetResponseMethod::HoldExport(set_response) => {
|
||||
set_response.update_created_ids(&mut response);
|
||||
}
|
||||
@@ -455,6 +470,13 @@ impl RequestHandler for Server {
|
||||
.await?
|
||||
.into()
|
||||
}
|
||||
// inbuxa: inbuxa:DlpSettings/get
|
||||
GetRequestMethod::DlpSettings(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
crate::inbuxa::dlp_settings::get(self, access_token, *req)
|
||||
.await?
|
||||
.into()
|
||||
}
|
||||
// inbuxa: inbuxa:DataInventory/get
|
||||
GetRequestMethod::DataInventory(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
@@ -486,6 +508,28 @@ impl RequestHandler for Server {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
crate::inbuxa::legal_hold::get(self, *req).await?.into()
|
||||
}
|
||||
// inbuxa: mail held for review
|
||||
GetRequestMethod::HeldMessage(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
crate::inbuxa::held_message::get(self, access_token, *req).await?.into()
|
||||
}
|
||||
// inbuxa: DLP and mail flow rules
|
||||
GetRequestMethod::MailRule(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
crate::inbuxa::mail_rule::get(self, access_token, *req).await?.into()
|
||||
}
|
||||
// inbuxa: accepted security to-do items
|
||||
GetRequestMethod::SecurityAcceptance(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
crate::inbuxa::security_acceptance::get(self, access_token, *req)
|
||||
.await?
|
||||
.into()
|
||||
}
|
||||
// inbuxa: journaling
|
||||
GetRequestMethod::Journal(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
crate::inbuxa::journal::get(self, access_token, *req).await?.into()
|
||||
}
|
||||
// inbuxa: the audit log (AU-9)
|
||||
GetRequestMethod::AuditEvent(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
@@ -810,6 +854,23 @@ impl RequestHandler for Server {
|
||||
.await?
|
||||
.into()
|
||||
}
|
||||
// inbuxa: inbuxa:DlpSettings/set
|
||||
SetRequestMethod::DlpSettings(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
// inbuxa: AU-1.2, AU-3
|
||||
crate::inbuxa::audit::recorded(
|
||||
self,
|
||||
access_token,
|
||||
session,
|
||||
&method_name.obj.to_string(),
|
||||
None,
|
||||
None,
|
||||
*req,
|
||||
|req| Box::pin(crate::inbuxa::dlp_settings::set(self, access_token, req)),
|
||||
)
|
||||
.await?
|
||||
.into()
|
||||
}
|
||||
// inbuxa: the audit log (AU-7, AU-11, AU-6)
|
||||
SetRequestMethod::AuditSettings(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
@@ -910,6 +971,77 @@ impl RequestHandler for Server {
|
||||
.await?
|
||||
.into()
|
||||
}
|
||||
SetRequestMethod::HeldMessage(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
let reason = req.arguments.reason.clone();
|
||||
crate::inbuxa::audit::recorded(
|
||||
self,
|
||||
access_token,
|
||||
session,
|
||||
&method_name.obj.to_string(),
|
||||
None,
|
||||
reason,
|
||||
*req,
|
||||
|req| Box::pin(crate::inbuxa::held_message::set(self, access_token, req)),
|
||||
)
|
||||
.await?
|
||||
.into()
|
||||
}
|
||||
SetRequestMethod::MailRule(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
let reason = req.arguments.reason.clone();
|
||||
crate::inbuxa::audit::recorded(
|
||||
self,
|
||||
access_token,
|
||||
session,
|
||||
&method_name.obj.to_string(),
|
||||
None,
|
||||
reason,
|
||||
*req,
|
||||
|req| Box::pin(crate::inbuxa::mail_rule::set(self, access_token, req)),
|
||||
)
|
||||
.await?
|
||||
.into()
|
||||
}
|
||||
// inbuxa: SS-26, every acceptance made or removed is in the
|
||||
// audit log
|
||||
SetRequestMethod::SecurityAcceptance(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
crate::inbuxa::audit::recorded(
|
||||
self,
|
||||
access_token,
|
||||
session,
|
||||
&method_name.obj.to_string(),
|
||||
None,
|
||||
None,
|
||||
*req,
|
||||
|req| {
|
||||
Box::pin(crate::inbuxa::security_acceptance::set(
|
||||
self,
|
||||
access_token,
|
||||
req,
|
||||
))
|
||||
},
|
||||
)
|
||||
.await?
|
||||
.into()
|
||||
}
|
||||
SetRequestMethod::Journal(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
let reason = req.arguments.reason.clone();
|
||||
crate::inbuxa::audit::recorded(
|
||||
self,
|
||||
access_token,
|
||||
session,
|
||||
&method_name.obj.to_string(),
|
||||
None,
|
||||
reason,
|
||||
*req,
|
||||
|req| Box::pin(crate::inbuxa::journal::set(self, access_token, req)),
|
||||
)
|
||||
.await?
|
||||
.into()
|
||||
}
|
||||
SetRequestMethod::AuditExport(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
crate::inbuxa::audit_log::export_set(self, access_token, session, *req)
|
||||
|
||||
@@ -419,6 +419,7 @@ impl IntermediateChangesResponse {
|
||||
| MethodObject::DeletedAccount
|
||||
| MethodObject::AiLimits
|
||||
| MethodObject::LogSettings
|
||||
| MethodObject::DlpSettings
|
||||
| MethodObject::DataInventory
|
||||
| MethodObject::InventorySnapshot
|
||||
| MethodObject::Explanation
|
||||
@@ -429,6 +430,10 @@ impl IntermediateChangesResponse {
|
||||
| MethodObject::AccountLock
|
||||
| MethodObject::LegalHold
|
||||
| MethodObject::HoldExport
|
||||
| MethodObject::MailRule
|
||||
| MethodObject::SecurityAcceptance
|
||||
| MethodObject::Journal
|
||||
| MethodObject::HeldMessage
|
||||
| MethodObject::ProtocolPolicy
|
||||
| MethodObject::TenantProtocolPolicy
|
||||
| MethodObject::Registry(_) => unreachable!(),
|
||||
|
||||
@@ -217,7 +217,11 @@ async fn before<T: JmapObject>(
|
||||
|
||||
if let Some(MaybeResultReference::Value(destroy)) = &request.destroy {
|
||||
for id in destroy {
|
||||
let before = stored(server, registry, id).await;
|
||||
// inbuxa: a fork object is named from its own store, as an update is
|
||||
let before = match registry {
|
||||
Some(_) => stored(server, registry, id).await,
|
||||
None => fork_current(server, object, id).await,
|
||||
};
|
||||
let mut described = before.as_ref().map(diff::describe).unwrap_or_default();
|
||||
if let Some(before) = &before {
|
||||
described.name = full_name(server, object, before, described.name).await;
|
||||
@@ -434,6 +438,26 @@ async fn fork_current(server: &Server, object: &str, id: &MaybeInvalid<Id>) -> O
|
||||
}
|
||||
MaybeInvalid::Invalid(_) => None,
|
||||
},
|
||||
// SS-26: an acceptance named by its check and subject
|
||||
"inbuxa:SecurityAcceptance" => match id {
|
||||
MaybeInvalid::Value(id) => {
|
||||
let acceptance =
|
||||
security::acceptance::get(data, u32::try_from(id.id()).ok()?)
|
||||
.await
|
||||
.ok()??;
|
||||
let name = match acceptance.subject.as_str() {
|
||||
"" => acceptance.check.clone(),
|
||||
subject => format!("{} {subject}", acceptance.check),
|
||||
};
|
||||
Some(serde_json::json!({
|
||||
"name": name,
|
||||
"check": acceptance.check,
|
||||
"subject": acceptance.subject,
|
||||
"note": acceptance.note,
|
||||
}))
|
||||
}
|
||||
MaybeInvalid::Invalid(_) => None,
|
||||
},
|
||||
"inbuxa:TenantProtocolPolicy" => match id {
|
||||
MaybeInvalid::Value(id) => {
|
||||
security::tenant_protocol_policy::get(data, id.document_id())
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! `POST /api/directory/test`: try a saved directory before anything signs in
|
||||
//! through it (settings-reorg, guided setup "Connect a sign-in directory").
|
||||
//!
|
||||
//! The body names a directory and an address, and optionally a password:
|
||||
//!
|
||||
//! ```json
|
||||
//! {"directoryId": "b", "address": "[email protected]", "password": "…"}
|
||||
//! ```
|
||||
//!
|
||||
//! The answer says whether the directory opened, what a recipient lookup of
|
||||
//! the address finds, and, when a password is given, whether it signs in.
|
||||
//! It calls the directory itself, below the sign-in path, so a test never
|
||||
//! creates or updates an account (DIR-14), never counts toward the sign-in
|
||||
//! ban, and doesn't mind which domains use the directory (DIR-6). Nothing is
|
||||
//! cached (DIR-32). A password hash a directory hands back is never returned.
|
||||
//!
|
||||
//! For server-level administrators who may change directories.
|
||||
|
||||
use common::{Server, auth::AccessToken};
|
||||
use directory::{Credentials, Directory, Recipient};
|
||||
use registry::schema::enums::Permission;
|
||||
use serde_json::{Value, json};
|
||||
use std::str::FromStr;
|
||||
use types::id::Id;
|
||||
|
||||
fn message(err: &trc::Error) -> String {
|
||||
err.value_as_str(trc::Key::Reason)
|
||||
.or_else(|| err.value_as_str(trc::Key::Details))
|
||||
.map(str::to_string)
|
||||
.unwrap_or_else(|| err.to_string())
|
||||
}
|
||||
|
||||
fn kind(directory: &Directory) -> &'static str {
|
||||
match directory {
|
||||
Directory::Ldap(_) => "ldap",
|
||||
Directory::Sql(_) => "sql",
|
||||
Directory::OpenId(_) => "oidc",
|
||||
Directory::Unavailable(d) => match d.directory_type() {
|
||||
registry::schema::enums::DirectoryType::Ldap => "ldap",
|
||||
registry::schema::enums::DirectoryType::Sql => "sql",
|
||||
registry::schema::enums::DirectoryType::Oidc => "oidc",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn assert_allowed(access_token: &AccessToken) -> trc::Result<()> {
|
||||
if access_token.tenant_id().is_some() {
|
||||
return Err(trc::JmapEvent::Forbidden
|
||||
.into_err()
|
||||
.details("Directory tests are for server-level administrators."));
|
||||
}
|
||||
access_token.enforce_permission(Permission::SysDirectoryUpdate)
|
||||
}
|
||||
|
||||
fn bad(details: &'static str) -> trc::Error {
|
||||
trc::ResourceEvent::BadParameters.into_err().details(details)
|
||||
}
|
||||
|
||||
pub async fn test(server: &Server, body: &Value) -> trc::Result<Value> {
|
||||
let directory_id = body
|
||||
.get("directoryId")
|
||||
.and_then(Value::as_str)
|
||||
.and_then(|id| Id::from_str(id).ok())
|
||||
.ok_or_else(|| bad("Expected {\"directoryId\": …, \"address\": …}"))?;
|
||||
let address = body
|
||||
.get("address")
|
||||
.and_then(Value::as_str)
|
||||
.map(|a| a.trim().to_lowercase())
|
||||
.filter(|a| !a.is_empty())
|
||||
.ok_or_else(|| bad("Expected an address to look up"))?;
|
||||
let password = body
|
||||
.get("password")
|
||||
.and_then(Value::as_str)
|
||||
.filter(|p| !p.is_empty());
|
||||
|
||||
let Some(directory) = server
|
||||
.core
|
||||
.storage
|
||||
.directories
|
||||
.get(&(directory_id.id() as u32))
|
||||
.cloned()
|
||||
else {
|
||||
return Ok(json!({
|
||||
"opened": false,
|
||||
"error": "The server hasn't loaded this directory. Save it, and try again in a few seconds.",
|
||||
}));
|
||||
};
|
||||
|
||||
let mut out = json!({ "kind": kind(&directory) });
|
||||
if let Directory::Unavailable(d) = directory.as_ref() {
|
||||
out["opened"] = json!(false);
|
||||
out["error"] = json!(message(&d.error()));
|
||||
return Ok(out);
|
||||
}
|
||||
out["opened"] = json!(true);
|
||||
|
||||
if let Some(discovery) = directory.oidc_discovery_document() {
|
||||
out["oidc"] = json!({
|
||||
"issuer": discovery.document.issuer,
|
||||
"jwksUri": discovery.document.jwks_uri,
|
||||
});
|
||||
}
|
||||
|
||||
// What mail for this address would find.
|
||||
if directory.can_lookup_recipients() {
|
||||
out["lookup"] = match directory.recipient(&address).await {
|
||||
Ok(Recipient::Account(a)) => json!({
|
||||
"found": "account",
|
||||
"email": a.email,
|
||||
"aliases": a.email_aliases,
|
||||
"groups": a.groups.unwrap_or_default(),
|
||||
"description": a.description,
|
||||
}),
|
||||
Ok(Recipient::Group(g)) => json!({
|
||||
"found": "group",
|
||||
"email": g.email,
|
||||
"aliases": g.email_aliases,
|
||||
"description": g.description,
|
||||
}),
|
||||
Ok(Recipient::Invalid) => json!({ "found": "none" }),
|
||||
Err(err) => json!({ "error": message(&err) }),
|
||||
};
|
||||
}
|
||||
|
||||
// Whether this person could sign in. OIDC takes tokens, not passwords
|
||||
// (DIR-29), so there's nothing to try there.
|
||||
if let Some(password) = password
|
||||
&& !matches!(directory.as_ref(), Directory::OpenId(_))
|
||||
{
|
||||
let credentials = Credentials::Basic {
|
||||
username: address.clone(),
|
||||
secret: password.to_string(),
|
||||
mfa_token: None,
|
||||
};
|
||||
out["signIn"] = match directory.authenticate(&credentials).await {
|
||||
Ok(a) => json!({
|
||||
"ok": true,
|
||||
"email": a.email,
|
||||
"groups": a.groups.unwrap_or_default(),
|
||||
"description": a.description,
|
||||
}),
|
||||
Err(err) if matches!(err.as_ref(), trc::EventType::Auth(trc::AuthEvent::Failed)) => {
|
||||
json!({ "ok": false, "wrongPassword": true })
|
||||
}
|
||||
Err(err) => json!({ "ok": false, "error": message(&err) }),
|
||||
};
|
||||
}
|
||||
|
||||
Ok(out)
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! `inbuxa:DlpSettings/get` and `/set`: how many days held mail waits for a
|
||||
//! reviewer (dlp-and-mail-flow-rules spec, §2.6), 1 to 90, 7 by default.
|
||||
//! Server-level, like the rules; applies to mail held from then on.
|
||||
|
||||
use common::{Server, auth::AccessToken};
|
||||
use inbuxa_features::mailflow::held::{self, Settings};
|
||||
use jmap_proto::{
|
||||
error::set::SetError,
|
||||
method::{
|
||||
get::{GetRequest, GetResponse},
|
||||
set::{SetRequest, SetResponse},
|
||||
},
|
||||
object::inbuxa_dlp_settings::{DlpSettings, DlpSettingsProperty as P, DlpSettingsValue},
|
||||
request::IntoValid,
|
||||
};
|
||||
use jmap_tools::{Key, Map, Value};
|
||||
use types::id::Id;
|
||||
|
||||
type LValue = Value<'static, P, DlpSettingsValue>;
|
||||
|
||||
const ALL: &[P] = &[P::Id, P::KeepHeldDays];
|
||||
|
||||
fn assert_server_level(access_token: &AccessToken) -> trc::Result<()> {
|
||||
if access_token.tenant_id().is_some() {
|
||||
Err(trc::JmapEvent::Forbidden
|
||||
.into_err()
|
||||
.details("DLP settings are server-level."))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn to_value(settings: &Settings, properties: &[P]) -> LValue {
|
||||
let mut out = Map::with_capacity(properties.len());
|
||||
for property in properties {
|
||||
let value = match property {
|
||||
P::Id => Value::Element(DlpSettingsValue::Id(Id::singleton())),
|
||||
P::KeepHeldDays => Value::Number(settings.keep_held_days.into()),
|
||||
};
|
||||
out.insert_unchecked(Key::Property(property.clone()), value);
|
||||
}
|
||||
Value::Object(out)
|
||||
}
|
||||
|
||||
/// `inbuxa:DlpSettings/get`.
|
||||
pub async fn get(
|
||||
server: &Server,
|
||||
access_token: &AccessToken,
|
||||
mut request: GetRequest<DlpSettings>,
|
||||
) -> trc::Result<GetResponse<DlpSettings>> {
|
||||
assert_server_level(access_token)?;
|
||||
let properties = request.unwrap_properties(ALL);
|
||||
let (ids, not_found) = request.unwrap_ids(1)?;
|
||||
let mut response = GetResponse {
|
||||
account_id: request.account_id.into(),
|
||||
state: None,
|
||||
list: Vec::new(),
|
||||
not_found,
|
||||
};
|
||||
let settings = held::settings(server.store()).await?;
|
||||
match ids {
|
||||
None => response.list.push(to_value(&settings, &properties)),
|
||||
Some(ids) => {
|
||||
for id in ids {
|
||||
if id.is_singleton() {
|
||||
response.list.push(to_value(&settings, &properties));
|
||||
} else {
|
||||
response.push_not_found(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
fn apply(
|
||||
settings: &mut Settings,
|
||||
property: &P,
|
||||
value: &Value<'_, P, DlpSettingsValue>,
|
||||
) -> Result<(), String> {
|
||||
match property {
|
||||
P::KeepHeldDays => {
|
||||
settings.keep_held_days = value
|
||||
.as_u64()
|
||||
.ok_or_else(|| "must be a whole number of days".to_string())?
|
||||
}
|
||||
P::Id => return Err("is immutable".to_string()),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `inbuxa:DlpSettings/set`: updates the singleton.
|
||||
pub async fn set(
|
||||
server: &Server,
|
||||
access_token: &AccessToken,
|
||||
mut request: SetRequest<'_, DlpSettings>,
|
||||
) -> trc::Result<SetResponse<DlpSettings>> {
|
||||
assert_server_level(access_token)?;
|
||||
let mut response = SetResponse::from_request(&request, server.core.jmap.set_max_objects)?;
|
||||
for (client_id, _) in request.unwrap_create() {
|
||||
response
|
||||
.not_created
|
||||
.append(client_id, SetError::singleton());
|
||||
}
|
||||
for id in request.unwrap_destroy().into_valid() {
|
||||
response.not_destroyed.append(id, SetError::singleton());
|
||||
}
|
||||
let data = server.store();
|
||||
for (id, value) in request.unwrap_update().into_valid() {
|
||||
if !id.is_singleton() {
|
||||
response.not_updated.append(id, SetError::not_found());
|
||||
continue;
|
||||
}
|
||||
let mut settings = held::settings(data).await?;
|
||||
let mut error = None;
|
||||
for (key, value) in value.into_expanded_object() {
|
||||
let Key::Property(property) = &key else {
|
||||
error = Some(SetError::invalid_properties().with_property(key.into_owned()));
|
||||
break;
|
||||
};
|
||||
if let Err(why) = apply(&mut settings, property, &value) {
|
||||
error = Some(
|
||||
SetError::invalid_properties()
|
||||
.with_property(property.clone())
|
||||
.with_description(why),
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if error.is_none()
|
||||
&& let Err((property, why)) = settings.check()
|
||||
{
|
||||
error = Some(
|
||||
SetError::invalid_properties()
|
||||
.with_property(property.parse::<P>().unwrap_or(P::Id))
|
||||
.with_description(format!("{property} {why}.")),
|
||||
);
|
||||
}
|
||||
match error {
|
||||
Some(error) => response.not_updated.append(id, error),
|
||||
None => {
|
||||
held::set_settings(data, &settings).await?;
|
||||
response.updated.append(id, None);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
@@ -798,6 +798,10 @@ mod tests {
|
||||
assert!(delivery_facts(&mut Facts::default(), &message, "[email protected]").is_err());
|
||||
}
|
||||
|
||||
fn is_timestamp(value: &str) -> bool {
|
||||
chrono::DateTime::parse_from_rfc3339(value).is_ok()
|
||||
}
|
||||
|
||||
/// The settings questions a release prepares answers for (EX-26): every
|
||||
/// non-secret property of every settings object, at the object's own
|
||||
/// default, built exactly as a live question is.
|
||||
@@ -842,6 +846,12 @@ mod tests {
|
||||
if info.secret {
|
||||
continue;
|
||||
}
|
||||
// A date's default is the moment the object is built, so its
|
||||
// question changes every run and no live question ever
|
||||
// matches it: nothing worth preparing.
|
||||
if matches!(map[&property].as_str(), Some(v) if is_timestamp(v)) {
|
||||
continue;
|
||||
}
|
||||
let mut facts = Facts::default();
|
||||
push_setting(&mut facts, &object, &property, &info, &map[&property]);
|
||||
out.push((object.clone(), property, facts));
|
||||
@@ -856,6 +866,7 @@ mod tests {
|
||||
assert!(questions.len() > 500, "found {}", questions.len());
|
||||
assert!(questions.iter().any(|(o, p, _)| o == "x:Domain" && p == "dnsManagement"));
|
||||
assert!(!questions.iter().any(|(o, p, _)| o == "x:AiModel" && p == "httpAuth"));
|
||||
assert!(!questions.iter().any(|(o, p, _)| o == "x:Account" && p == "createdAt"));
|
||||
}
|
||||
|
||||
/// Writes `resources/explain/settings.json.gz` (EX-26). Run before a
|
||||
|
||||
@@ -0,0 +1,325 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! `inbuxa:HeldMessage` (dlp-and-mail-flow-rules spec, §2.6, §2.8): the
|
||||
//! review queue. `sysDlpReviewGet` lists held mail and reads it;
|
||||
//! `sysDlpReviewUpdate` releases or rejects it, with a reason the request
|
||||
//! layer records. Reading a held message's text is recorded as access to
|
||||
//! the sender's mail. Nobody in a tenant reaches this (settled answer 3).
|
||||
|
||||
use common::{Server, auth::AccessToken, config::smtp::queue::QueueName};
|
||||
use inbuxa_features::{
|
||||
audit::{Action, Outcome, Record, Target},
|
||||
mailflow::held::{self, Held},
|
||||
};
|
||||
use jmap_proto::{
|
||||
error::set::SetError,
|
||||
method::{
|
||||
get::{GetRequest, GetResponse},
|
||||
set::{SetRequest, SetResponse},
|
||||
},
|
||||
object::inbuxa_held_message::{
|
||||
HeldMessage, HeldMessageProperty as P, HeldMessageSetArguments, HeldMessageValue,
|
||||
},
|
||||
request::IntoValid,
|
||||
types::date::UTCDate,
|
||||
};
|
||||
use jmap_tools::{Key, Map, Value};
|
||||
use mail_parser::{MessageParser, MimeHeaders, PartType};
|
||||
use smtp::queue::spool::SmtpSpool;
|
||||
use std::borrow::Cow;
|
||||
use types::id::Id;
|
||||
|
||||
type HValue = Value<'static, P, HeldMessageValue>;
|
||||
|
||||
const ALL: &[P] = &[
|
||||
P::Id,
|
||||
P::Sender,
|
||||
P::Recipients,
|
||||
P::Subject,
|
||||
P::Size,
|
||||
P::Rules,
|
||||
P::Counts,
|
||||
P::HeldAt,
|
||||
P::ExpiresAt,
|
||||
];
|
||||
|
||||
/// How much of a held message's text a preview shows.
|
||||
const PREVIEW_LIMIT: usize = 64 * 1024;
|
||||
|
||||
fn server_level(access_token: &AccessToken) -> trc::Result<()> {
|
||||
if access_token.tenant_id().is_some() {
|
||||
Err(trc::JmapEvent::Forbidden
|
||||
.into_err()
|
||||
.details("Held mail is the server's to review."))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn date(seconds: u64) -> HValue {
|
||||
Value::Str(UTCDate::from_timestamp(seconds as i64).to_string().into())
|
||||
}
|
||||
|
||||
fn text(s: &str) -> HValue {
|
||||
Value::Str(Cow::Owned(s.to_string()))
|
||||
}
|
||||
|
||||
/// The text a reviewer reads: the subject, each body as text, and the
|
||||
/// attachments' names; at most [`PREVIEW_LIMIT`].
|
||||
async fn preview(server: &Server, queue_id: u64) -> trc::Result<Option<String>> {
|
||||
let Some(message) = server.read_message(queue_id, QueueName::default()).await else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(raw) = server
|
||||
.blob_store()
|
||||
.get_blob(message.message.blob_hash.as_slice(), 0..usize::MAX)
|
||||
.await?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(parsed) = MessageParser::new().parse(&raw) else {
|
||||
return Ok(Some(
|
||||
String::from_utf8_lossy(&raw[..raw.len().min(PREVIEW_LIMIT)]).into_owned(),
|
||||
));
|
||||
};
|
||||
let mut out = String::new();
|
||||
for part in parsed.text_bodies() {
|
||||
match &part.body {
|
||||
PartType::Text(text) => out.push_str(text),
|
||||
PartType::Html(html) => out.push_str(&mail_parser::decoders::html::html_to_text(html)),
|
||||
_ => {}
|
||||
}
|
||||
out.push_str("\n\n");
|
||||
}
|
||||
let attachments: Vec<&str> = parsed
|
||||
.attachments()
|
||||
.filter_map(|a| a.attachment_name())
|
||||
.collect();
|
||||
if !attachments.is_empty() {
|
||||
out.push_str(&format!("Attachments: {}\n", attachments.join(", ")));
|
||||
}
|
||||
if out.len() > PREVIEW_LIMIT {
|
||||
let mut cut = PREVIEW_LIMIT;
|
||||
while !out.is_char_boundary(cut) {
|
||||
cut -= 1;
|
||||
}
|
||||
out.truncate(cut);
|
||||
}
|
||||
Ok(Some(out))
|
||||
}
|
||||
|
||||
fn to_value(record: &Held, properties: &[P], preview: Option<&str>) -> HValue {
|
||||
let mut out = Map::with_capacity(properties.len());
|
||||
for property in properties {
|
||||
let value = match property {
|
||||
P::Id => Value::Element(HeldMessageValue::Id(Id::from(record.queue_id))),
|
||||
P::Sender => text(&record.sender),
|
||||
P::Recipients => Value::Array(record.recipients.iter().map(|r| text(r)).collect()),
|
||||
P::Subject => text(&record.subject),
|
||||
P::Size => Value::Number(record.size.into()),
|
||||
P::Rules => Value::Array(
|
||||
record
|
||||
.rules
|
||||
.iter()
|
||||
.map(|rule| {
|
||||
let mut map = Map::with_capacity(2);
|
||||
map.insert_unchecked(Key::Borrowed("name"), text(&rule.name));
|
||||
map.insert_unchecked(Key::Borrowed("notice"), text(&rule.notice));
|
||||
Value::Object(map)
|
||||
})
|
||||
.collect(),
|
||||
),
|
||||
P::Counts => Value::Array(
|
||||
record
|
||||
.counts
|
||||
.iter()
|
||||
.map(|(detector, count)| {
|
||||
let mut map = Map::with_capacity(2);
|
||||
map.insert_unchecked(Key::Borrowed("detector"), text(detector));
|
||||
map.insert_unchecked(
|
||||
Key::Borrowed("count"),
|
||||
Value::Number((*count as u64).into()),
|
||||
);
|
||||
Value::Object(map)
|
||||
})
|
||||
.collect(),
|
||||
),
|
||||
P::HeldAt => date(record.held_at),
|
||||
P::ExpiresAt => date(record.expires_at),
|
||||
P::Preview => preview.map_or(Value::Null, text),
|
||||
P::Decision | P::Note => Value::Null,
|
||||
};
|
||||
out.insert_unchecked(Key::Property(property.clone()), value);
|
||||
}
|
||||
Value::Object(out)
|
||||
}
|
||||
|
||||
/// `inbuxa:HeldMessage/get`: held mail, oldest first.
|
||||
pub async fn get(
|
||||
server: &Server,
|
||||
access_token: &AccessToken,
|
||||
mut request: GetRequest<HeldMessage>,
|
||||
) -> trc::Result<GetResponse<HeldMessage>> {
|
||||
server_level(access_token)?;
|
||||
let properties = request.unwrap_properties(ALL);
|
||||
let (ids, not_found) = request.unwrap_ids(server.core.jmap.get_max_objects)?;
|
||||
let mut response = GetResponse {
|
||||
account_id: request.account_id.into(),
|
||||
state: None,
|
||||
list: Vec::new(),
|
||||
not_found,
|
||||
};
|
||||
let all = held::all(server.store()).await?;
|
||||
let wanted: Vec<&Held> = match &ids {
|
||||
None => all.iter().collect(),
|
||||
Some(ids) => {
|
||||
let mut found = Vec::new();
|
||||
for id in ids {
|
||||
match all.iter().find(|h| h.queue_id == id.id()) {
|
||||
Some(record) => found.push(record),
|
||||
None => response.push_not_found(*id),
|
||||
}
|
||||
}
|
||||
found
|
||||
}
|
||||
};
|
||||
let with_preview = properties.contains(&P::Preview);
|
||||
for record in wanted {
|
||||
let text = if with_preview {
|
||||
let text = preview(server, record.queue_id).await?;
|
||||
// Reading someone's mail is recorded, as any access is
|
||||
server
|
||||
.audit_note(Record {
|
||||
at: store::write::now() * 1000,
|
||||
actor: server.audit_actor(access_token).await,
|
||||
via: access_token.origin().cloned(),
|
||||
remote_ip: None,
|
||||
action: Action::BlobAccess,
|
||||
target: Target {
|
||||
kind: "inbuxa:HeldMessage".into(),
|
||||
id: Some(Id::from(record.queue_id).to_string()),
|
||||
name: Some(record.subject.clone()),
|
||||
account_id: record.account_id,
|
||||
tenant_id: record.tenant_id,
|
||||
},
|
||||
changes: vec![],
|
||||
details: Some(format!(
|
||||
"Read a message held for review, from {}",
|
||||
record.sender
|
||||
)),
|
||||
reason: None,
|
||||
outcome: Outcome::success(),
|
||||
})
|
||||
.await;
|
||||
text
|
||||
} else {
|
||||
None
|
||||
};
|
||||
response
|
||||
.list
|
||||
.push(to_value(record, &properties, text.as_deref()));
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
fn invalid(property: P, why: &str) -> SetError<P> {
|
||||
SetError::invalid_properties()
|
||||
.with_property(property)
|
||||
.with_description(why.to_string())
|
||||
}
|
||||
|
||||
/// `inbuxa:HeldMessage/set`: update with `decision` release or reject (and
|
||||
/// an optional `note` for the sender). There is no create or destroy.
|
||||
pub async fn set(
|
||||
server: &Server,
|
||||
access_token: &AccessToken,
|
||||
mut request: SetRequest<'_, HeldMessage>,
|
||||
) -> trc::Result<SetResponse<HeldMessage>> {
|
||||
server_level(access_token)?;
|
||||
let mut response = SetResponse::from_request(&request, server.core.jmap.set_max_objects)?;
|
||||
let arguments: HeldMessageSetArguments = std::mem::take(&mut request.arguments);
|
||||
let has_reason = arguments
|
||||
.reason
|
||||
.as_deref()
|
||||
.is_some_and(|r| !r.trim().is_empty());
|
||||
|
||||
for (client_id, _) in request.unwrap_create() {
|
||||
response.not_created.append(
|
||||
client_id,
|
||||
SetError::forbidden().with_description("Mail is held by DLP rules, not created."),
|
||||
);
|
||||
}
|
||||
|
||||
'update: for (id, value) in request.unwrap_update().into_valid() {
|
||||
let Some(record) = held::get(server.store(), id.id()).await? else {
|
||||
response.not_updated.append(id, SetError::not_found());
|
||||
continue;
|
||||
};
|
||||
if !has_reason {
|
||||
response.not_updated.append(
|
||||
id,
|
||||
SetError::invalid_properties().with_description(
|
||||
"Say why: a reason is required and is kept in the audit log.",
|
||||
),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
let mut decision = None;
|
||||
let mut note = None;
|
||||
for (key, value) in value.into_expanded_object() {
|
||||
match (&key, value) {
|
||||
(Key::Property(P::Decision), Value::Str(s)) if s == "release" || s == "reject" => {
|
||||
decision = Some(s.to_string());
|
||||
}
|
||||
(Key::Property(P::Note), Value::Str(s)) => {
|
||||
let s = s.trim();
|
||||
if !s.is_empty() {
|
||||
note = Some(s.chars().take(1000).collect::<String>());
|
||||
}
|
||||
}
|
||||
(Key::Property(P::Note), Value::Null) => {}
|
||||
_ => {
|
||||
response.not_updated.append(
|
||||
id,
|
||||
invalid(
|
||||
P::Decision,
|
||||
"Send decision: \"release\" or \"reject\", and an optional note.",
|
||||
),
|
||||
);
|
||||
continue 'update;
|
||||
}
|
||||
}
|
||||
}
|
||||
let done = match decision.as_deref() {
|
||||
Some("release") => smtp::queue::held::release(server, record.queue_id).await?,
|
||||
Some("reject") => smtp::queue::held::reject(server, &record, note.as_deref()).await?,
|
||||
_ => {
|
||||
response
|
||||
.not_updated
|
||||
.append(id, invalid(P::Decision, "Say release or reject."));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if done {
|
||||
response.updated.append(id, None);
|
||||
} else {
|
||||
response.not_updated.append(
|
||||
id,
|
||||
SetError::not_found().with_description("The message is no longer in the queue."),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for id in request.unwrap_destroy().into_valid() {
|
||||
response.not_destroyed.append(
|
||||
id,
|
||||
SetError::forbidden().with_description("Release or reject it instead."),
|
||||
);
|
||||
}
|
||||
|
||||
Ok(response)
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! `inbuxa:Journal` (journaling spec, JR-9, JR-12, JR-18): journals, seen
|
||||
//! with `sysJournalGet` and changed with `sysJournalUpdate`, which the
|
||||
//! request layer checks. Journals are the server's: nobody in a tenant
|
||||
//! reaches them. The request layer records every change in the audit log.
|
||||
//! Changing or removing a journal never touches what it has taken.
|
||||
|
||||
use common::{Server, auth::AccessToken};
|
||||
use inbuxa_features::journal::{self, Journal as Stored};
|
||||
use jmap_proto::{
|
||||
error::set::SetError,
|
||||
method::{
|
||||
get::{GetRequest, GetResponse},
|
||||
set::{SetRequest, SetResponse},
|
||||
},
|
||||
object::inbuxa_journal::{Journal, JournalProperty as P, JournalValue},
|
||||
request::IntoValid,
|
||||
types::date::UTCDate,
|
||||
};
|
||||
use jmap_tools::{Key, Map, Property, Value};
|
||||
use std::borrow::Cow;
|
||||
use store::write::now;
|
||||
use types::id::Id;
|
||||
|
||||
type JValue = Value<'static, P, JournalValue>;
|
||||
|
||||
const ALL: &[P] = &[
|
||||
P::Id,
|
||||
P::Name,
|
||||
P::Description,
|
||||
P::Enabled,
|
||||
P::Direction,
|
||||
P::Scope,
|
||||
P::RetentionDays,
|
||||
P::CreatedBy,
|
||||
P::CreatedAt,
|
||||
P::UpdatedAt,
|
||||
];
|
||||
|
||||
/// Properties the server sets; a client that sends them is refused.
|
||||
const SERVER_SET: &[P] = &[P::Id, P::CreatedBy, P::CreatedAt, P::UpdatedAt];
|
||||
|
||||
fn server_level(access_token: &AccessToken) -> trc::Result<()> {
|
||||
if access_token.tenant_id().is_some() {
|
||||
Err(trc::JmapEvent::Forbidden
|
||||
.into_err()
|
||||
.details("Journals are the server's."))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn json_to_value(json: serde_json::Value) -> JValue {
|
||||
match json {
|
||||
serde_json::Value::Null => Value::Null,
|
||||
serde_json::Value::Bool(b) => Value::Bool(b),
|
||||
serde_json::Value::Number(n) => {
|
||||
if let Some(n) = n.as_u64() {
|
||||
Value::Number(n.into())
|
||||
} else if let Some(n) = n.as_i64() {
|
||||
Value::Number(n.into())
|
||||
} else {
|
||||
Value::Number(n.as_f64().unwrap_or_default().into())
|
||||
}
|
||||
}
|
||||
serde_json::Value::String(s) => Value::Str(Cow::Owned(s)),
|
||||
serde_json::Value::Array(items) => {
|
||||
Value::Array(items.into_iter().map(json_to_value).collect())
|
||||
}
|
||||
serde_json::Value::Object(map) => {
|
||||
let mut out = Map::with_capacity(map.len());
|
||||
for (key, value) in map {
|
||||
out.insert_unchecked(Key::Owned(key), json_to_value(value));
|
||||
}
|
||||
Value::Object(out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn date(seconds: u64) -> JValue {
|
||||
Value::Str(UTCDate::from_timestamp(seconds as i64).to_string().into())
|
||||
}
|
||||
|
||||
fn to_value(journal: &Stored, properties: &[P]) -> JValue {
|
||||
let json = serde_json::to_value(journal).unwrap_or_default();
|
||||
let mut out = Map::with_capacity(properties.len());
|
||||
for property in properties {
|
||||
let value = match property {
|
||||
P::Id => Value::Element(JournalValue::Id(Id::from(journal.id))),
|
||||
P::CreatedAt => date(journal.created_at),
|
||||
P::UpdatedAt => date(journal.updated_at),
|
||||
other => json
|
||||
.get(other.to_cow().as_ref())
|
||||
.cloned()
|
||||
.map_or(Value::Null, json_to_value),
|
||||
};
|
||||
out.insert_unchecked(Key::Property(property.clone()), value);
|
||||
}
|
||||
Value::Object(out)
|
||||
}
|
||||
|
||||
/// A journal as sent: its JSON object, top-level keys only those a client
|
||||
/// may set.
|
||||
fn client_json(
|
||||
value: Value<'_, P, JournalValue>,
|
||||
) -> Result<serde_json::Map<String, serde_json::Value>, SetError<P>> {
|
||||
let mut map = serde_json::Map::new();
|
||||
for (key, value) in value.into_expanded_object() {
|
||||
match &key {
|
||||
Key::Property(p) if SERVER_SET.contains(p) => {
|
||||
return Err(SetError::invalid_properties()
|
||||
.with_property(p.clone())
|
||||
.with_description("The server sets this."));
|
||||
}
|
||||
Key::Property(p) => {
|
||||
map.insert(p.to_cow().into_owned(), value.into());
|
||||
}
|
||||
_ => {
|
||||
return Err(SetError::invalid_properties().with_property(key.clone().into_owned()));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(map)
|
||||
}
|
||||
|
||||
fn parse(json: serde_json::Map<String, serde_json::Value>) -> Result<Stored, SetError<P>> {
|
||||
let journal: Stored =
|
||||
serde_json::from_value(serde_json::Value::Object(json)).map_err(|err| {
|
||||
SetError::invalid_properties().with_description(format!("Not a valid journal: {err}"))
|
||||
})?;
|
||||
journal.validate().map_err(|invalid| {
|
||||
let property = invalid.property.parse::<P>().unwrap_or(P::Name);
|
||||
SetError::invalid_properties()
|
||||
.with_property(property)
|
||||
.with_description(invalid.reason)
|
||||
})?;
|
||||
Ok(journal)
|
||||
}
|
||||
|
||||
fn journal_id(id: Id) -> Option<u32> {
|
||||
u32::try_from(id.id()).ok()
|
||||
}
|
||||
|
||||
/// `inbuxa:Journal/get`: every journal, oldest first.
|
||||
pub async fn get(
|
||||
server: &Server,
|
||||
access_token: &AccessToken,
|
||||
mut request: GetRequest<Journal>,
|
||||
) -> trc::Result<GetResponse<Journal>> {
|
||||
server_level(access_token)?;
|
||||
let properties = request.unwrap_properties(ALL);
|
||||
let (ids, not_found) = request.unwrap_ids(server.core.jmap.get_max_objects)?;
|
||||
let mut response = GetResponse {
|
||||
account_id: request.account_id.into(),
|
||||
state: None,
|
||||
list: Vec::new(),
|
||||
not_found,
|
||||
};
|
||||
let journals = journal::all(server.store()).await?;
|
||||
match ids {
|
||||
None => {
|
||||
response.list = journals
|
||||
.iter()
|
||||
.map(|journal| to_value(journal, &properties))
|
||||
.collect()
|
||||
}
|
||||
Some(ids) => {
|
||||
for id in ids {
|
||||
match journal_id(id).and_then(|id| journals.iter().find(|j| j.id == id)) {
|
||||
Some(journal) => response.list.push(to_value(journal, &properties)),
|
||||
None => response.push_not_found(id),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
/// `inbuxa:Journal/set`: create, change or remove journals.
|
||||
pub async fn set(
|
||||
server: &Server,
|
||||
access_token: &AccessToken,
|
||||
mut request: SetRequest<'_, Journal>,
|
||||
) -> trc::Result<SetResponse<Journal>> {
|
||||
server_level(access_token)?;
|
||||
let mut response = SetResponse::from_request(&request, server.core.jmap.set_max_objects)?;
|
||||
let data = server.store();
|
||||
let actor = server.audit_actor(access_token).await;
|
||||
|
||||
for (client_id, value) in request.unwrap_create() {
|
||||
let stored = match client_json(value).and_then(parse) {
|
||||
Ok(stored) => stored,
|
||||
Err(error) => {
|
||||
response.not_created.append(client_id, error);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let at = now();
|
||||
let stored = Stored {
|
||||
created_by: actor.name.clone(),
|
||||
created_at: at,
|
||||
updated_at: at,
|
||||
..stored
|
||||
};
|
||||
let id = journal::create(data, &stored).await?;
|
||||
let mut out = Map::with_capacity(1);
|
||||
out.insert_unchecked(
|
||||
Key::Property(P::Id),
|
||||
Value::Element(JournalValue::Id(Id::from(id))),
|
||||
);
|
||||
response.created.insert(client_id, Value::Object(out));
|
||||
}
|
||||
|
||||
for (id, value) in request.unwrap_update().into_valid() {
|
||||
let Some(current) = (match journal_id(id) {
|
||||
Some(journal_id) => journal::get(data, journal_id).await?,
|
||||
None => None,
|
||||
}) else {
|
||||
response.not_updated.append(id, SetError::not_found());
|
||||
continue;
|
||||
};
|
||||
// The stored journal, with each property sent replacing its own
|
||||
let mut json = match serde_json::to_value(¤t) {
|
||||
Ok(serde_json::Value::Object(map)) => map,
|
||||
_ => serde_json::Map::new(),
|
||||
};
|
||||
let changes = match client_json(value) {
|
||||
Ok(changes) => changes,
|
||||
Err(error) => {
|
||||
response.not_updated.append(id, error);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
json.extend(changes);
|
||||
let next = match parse(json) {
|
||||
Ok(next) => next,
|
||||
Err(error) => {
|
||||
response.not_updated.append(id, error);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let next = Stored {
|
||||
id: current.id,
|
||||
created_by: current.created_by.clone(),
|
||||
created_at: current.created_at,
|
||||
updated_at: now(),
|
||||
..next
|
||||
};
|
||||
if next != current {
|
||||
journal::update(data, &next).await?;
|
||||
}
|
||||
response.updated.append(id, None);
|
||||
}
|
||||
|
||||
for id in request.unwrap_destroy().into_valid() {
|
||||
let Some(current) = (match journal_id(id) {
|
||||
Some(journal_id) => journal::get(data, journal_id).await?,
|
||||
None => None,
|
||||
}) else {
|
||||
response.not_destroyed.append(id, SetError::not_found());
|
||||
continue;
|
||||
};
|
||||
journal::delete(data, current.id).await?;
|
||||
response.destroyed.push(id);
|
||||
}
|
||||
|
||||
Ok(response)
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! `inbuxa:MailRule` (dlp-and-mail-flow-rules spec, §2.2, §2.8): mail flow
|
||||
//! rules and DLP rules. One object, two kinds, each with its own
|
||||
//! permissions: `sysMailRuleGet`/`Update` for transport rules,
|
||||
//! `sysDlpPolicyGet`/`Update` for DLP rules. Rules are the server's: nobody
|
||||
//! in a tenant reaches them (settled answer 3). The request layer records
|
||||
//! every change in the audit log.
|
||||
|
||||
use common::{Server, auth::AccessToken};
|
||||
use inbuxa_features::mailflow::rules::{self, Kind, Rule};
|
||||
use jmap_proto::{
|
||||
error::set::SetError,
|
||||
method::{
|
||||
get::{GetRequest, GetResponse},
|
||||
set::{SetRequest, SetResponse},
|
||||
},
|
||||
object::inbuxa_mail_rule::{MailRule, MailRuleProperty as P, MailRuleValue},
|
||||
request::IntoValid,
|
||||
types::date::UTCDate,
|
||||
};
|
||||
use jmap_tools::{Key, Map, Property, Value};
|
||||
use registry::schema::enums::Permission;
|
||||
use std::borrow::Cow;
|
||||
use store::write::now;
|
||||
use types::id::Id;
|
||||
|
||||
type RValue = Value<'static, P, MailRuleValue>;
|
||||
|
||||
const ALL: &[P] = &[
|
||||
P::Id,
|
||||
P::Name,
|
||||
P::Description,
|
||||
P::Kind,
|
||||
P::Enabled,
|
||||
P::Priority,
|
||||
P::Direction,
|
||||
P::Conditions,
|
||||
P::Exceptions,
|
||||
P::Actions,
|
||||
P::StopProcessing,
|
||||
P::CreatedBy,
|
||||
P::CreatedAt,
|
||||
P::UpdatedAt,
|
||||
];
|
||||
|
||||
/// Properties the server sets; a client that sends them is refused.
|
||||
const SERVER_SET: &[P] = &[P::Id, P::CreatedBy, P::CreatedAt, P::UpdatedAt];
|
||||
|
||||
fn can_see(access_token: &AccessToken, kind: Kind) -> bool {
|
||||
access_token.has_permission(match kind {
|
||||
Kind::Dlp => Permission::SysDlpPolicyGet,
|
||||
Kind::Transport => Permission::SysMailRuleGet,
|
||||
})
|
||||
}
|
||||
|
||||
fn can_change(access_token: &AccessToken, kind: Kind) -> bool {
|
||||
access_token.has_permission(match kind {
|
||||
Kind::Dlp => Permission::SysDlpPolicyUpdate,
|
||||
Kind::Transport => Permission::SysMailRuleUpdate,
|
||||
})
|
||||
}
|
||||
|
||||
fn server_level(access_token: &AccessToken) -> trc::Result<()> {
|
||||
if access_token.tenant_id().is_some() {
|
||||
Err(trc::JmapEvent::Forbidden
|
||||
.into_err()
|
||||
.details("Mail rules are the server's."))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn json_to_value(json: serde_json::Value) -> RValue {
|
||||
match json {
|
||||
serde_json::Value::Null => Value::Null,
|
||||
serde_json::Value::Bool(b) => Value::Bool(b),
|
||||
serde_json::Value::Number(n) => {
|
||||
if let Some(n) = n.as_u64() {
|
||||
Value::Number(n.into())
|
||||
} else if let Some(n) = n.as_i64() {
|
||||
Value::Number(n.into())
|
||||
} else {
|
||||
Value::Number(n.as_f64().unwrap_or_default().into())
|
||||
}
|
||||
}
|
||||
serde_json::Value::String(s) => Value::Str(Cow::Owned(s)),
|
||||
serde_json::Value::Array(items) => {
|
||||
Value::Array(items.into_iter().map(json_to_value).collect())
|
||||
}
|
||||
serde_json::Value::Object(map) => {
|
||||
let mut out = Map::with_capacity(map.len());
|
||||
for (key, value) in map {
|
||||
out.insert_unchecked(Key::Owned(key), json_to_value(value));
|
||||
}
|
||||
Value::Object(out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn date(seconds: u64) -> RValue {
|
||||
Value::Str(UTCDate::from_timestamp(seconds as i64).to_string().into())
|
||||
}
|
||||
|
||||
fn to_value(rule: &Rule, properties: &[P]) -> RValue {
|
||||
let json = serde_json::to_value(rule).unwrap_or_default();
|
||||
let mut out = Map::with_capacity(properties.len());
|
||||
for property in properties {
|
||||
let value = match property {
|
||||
P::Id => Value::Element(MailRuleValue::Id(Id::from(rule.id))),
|
||||
P::CreatedAt => date(rule.created_at),
|
||||
P::UpdatedAt => date(rule.updated_at),
|
||||
other => json
|
||||
.get(other.to_cow().as_ref())
|
||||
.cloned()
|
||||
.map_or(Value::Null, json_to_value),
|
||||
};
|
||||
out.insert_unchecked(Key::Property(property.clone()), value);
|
||||
}
|
||||
Value::Object(out)
|
||||
}
|
||||
|
||||
/// A rule as sent: its JSON object, top-level keys only those a client may
|
||||
/// set.
|
||||
fn client_json(value: Value<'_, P, MailRuleValue>) -> Result<serde_json::Map<String, serde_json::Value>, SetError<P>> {
|
||||
let mut map = serde_json::Map::new();
|
||||
for (key, value) in value.into_expanded_object() {
|
||||
match &key {
|
||||
Key::Property(p) if SERVER_SET.contains(p) => {
|
||||
return Err(SetError::invalid_properties()
|
||||
.with_property(p.clone())
|
||||
.with_description("The server sets this."));
|
||||
}
|
||||
Key::Property(p) => {
|
||||
map.insert(p.to_cow().into_owned(), value.into());
|
||||
}
|
||||
_ => {
|
||||
return Err(SetError::invalid_properties().with_property(key.clone().into_owned()));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(map)
|
||||
}
|
||||
|
||||
fn parse(json: serde_json::Map<String, serde_json::Value>) -> Result<Rule, SetError<P>> {
|
||||
let rule: Rule = serde_json::from_value(serde_json::Value::Object(json)).map_err(|err| {
|
||||
SetError::invalid_properties().with_description(format!("Not a valid rule: {err}"))
|
||||
})?;
|
||||
rule.validate().map_err(|invalid| {
|
||||
let property = invalid.property.parse::<P>().unwrap_or(P::Name);
|
||||
SetError::invalid_properties()
|
||||
.with_property(property)
|
||||
.with_description(invalid.reason)
|
||||
})?;
|
||||
Ok(rule)
|
||||
}
|
||||
|
||||
fn forbidden(kind: Kind) -> SetError<P> {
|
||||
SetError::forbidden().with_description(match kind {
|
||||
Kind::Dlp => "Changing DLP rules needs the permission to change DLP rules.",
|
||||
Kind::Transport => "Changing mail flow rules needs the permission to change them.",
|
||||
})
|
||||
}
|
||||
|
||||
fn rule_id(id: Id) -> Option<u32> {
|
||||
u32::try_from(id.id()).ok()
|
||||
}
|
||||
|
||||
/// `inbuxa:MailRule/get`: the rules the caller may see, in the order they
|
||||
/// run.
|
||||
pub async fn get(
|
||||
server: &Server,
|
||||
access_token: &AccessToken,
|
||||
mut request: GetRequest<MailRule>,
|
||||
) -> trc::Result<GetResponse<MailRule>> {
|
||||
server_level(access_token)?;
|
||||
let properties = request.unwrap_properties(ALL);
|
||||
let (ids, not_found) = request.unwrap_ids(server.core.jmap.get_max_objects)?;
|
||||
let mut response = GetResponse {
|
||||
account_id: request.account_id.into(),
|
||||
state: None,
|
||||
list: Vec::new(),
|
||||
not_found,
|
||||
};
|
||||
let visible: Vec<Rule> = rules::all(server.store())
|
||||
.await?
|
||||
.into_iter()
|
||||
.filter(|rule| can_see(access_token, rule.kind))
|
||||
.collect();
|
||||
match ids {
|
||||
None => {
|
||||
response.list = visible
|
||||
.iter()
|
||||
.map(|rule| to_value(rule, &properties))
|
||||
.collect()
|
||||
}
|
||||
Some(ids) => {
|
||||
for id in ids {
|
||||
match rule_id(id).and_then(|id| visible.iter().find(|r| r.id == id)) {
|
||||
Some(rule) => response.list.push(to_value(rule, &properties)),
|
||||
None => response.push_not_found(id),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
/// `inbuxa:MailRule/set`: create, change or delete rules, each checked
|
||||
/// against the permissions for its kind (and, on a change of kind, both).
|
||||
pub async fn set(
|
||||
server: &Server,
|
||||
access_token: &AccessToken,
|
||||
mut request: SetRequest<'_, MailRule>,
|
||||
) -> trc::Result<SetResponse<MailRule>> {
|
||||
server_level(access_token)?;
|
||||
let mut response = SetResponse::from_request(&request, server.core.jmap.set_max_objects)?;
|
||||
let data = server.store();
|
||||
let actor = server.audit_actor(access_token).await;
|
||||
|
||||
for (client_id, value) in request.unwrap_create() {
|
||||
let rule = match client_json(value).and_then(parse) {
|
||||
Ok(rule) => rule,
|
||||
Err(error) => {
|
||||
response.not_created.append(client_id, error);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if !can_change(access_token, rule.kind) {
|
||||
response.not_created.append(client_id, forbidden(rule.kind));
|
||||
continue;
|
||||
}
|
||||
let at = now();
|
||||
let rule = Rule {
|
||||
created_by: actor.name.clone(),
|
||||
created_at: at,
|
||||
updated_at: at,
|
||||
..rule
|
||||
};
|
||||
let id = rules::create(data, &rule).await?;
|
||||
let mut out = Map::with_capacity(1);
|
||||
out.insert_unchecked(
|
||||
Key::Property(P::Id),
|
||||
Value::Element(MailRuleValue::Id(Id::from(id))),
|
||||
);
|
||||
response.created.insert(client_id, Value::Object(out));
|
||||
}
|
||||
|
||||
for (id, value) in request.unwrap_update().into_valid() {
|
||||
let Some(current) = (match rule_id(id) {
|
||||
Some(rule_id) => rules::get(data, rule_id).await?,
|
||||
None => None,
|
||||
}) else {
|
||||
response.not_updated.append(id, SetError::not_found());
|
||||
continue;
|
||||
};
|
||||
if !can_see(access_token, current.kind) {
|
||||
response.not_updated.append(id, SetError::not_found());
|
||||
continue;
|
||||
}
|
||||
if !can_change(access_token, current.kind) {
|
||||
response.not_updated.append(id, forbidden(current.kind));
|
||||
continue;
|
||||
}
|
||||
// The stored rule, with each property sent replacing its own
|
||||
let mut json = match serde_json::to_value(¤t) {
|
||||
Ok(serde_json::Value::Object(map)) => map,
|
||||
_ => serde_json::Map::new(),
|
||||
};
|
||||
let changes = match client_json(value) {
|
||||
Ok(changes) => changes,
|
||||
Err(error) => {
|
||||
response.not_updated.append(id, error);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
json.extend(changes);
|
||||
let next = match parse(json) {
|
||||
Ok(next) => next,
|
||||
Err(error) => {
|
||||
response.not_updated.append(id, error);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if next.kind != current.kind && !can_change(access_token, next.kind) {
|
||||
response.not_updated.append(id, forbidden(next.kind));
|
||||
continue;
|
||||
}
|
||||
let next = Rule {
|
||||
id: current.id,
|
||||
created_by: current.created_by.clone(),
|
||||
created_at: current.created_at,
|
||||
updated_at: now(),
|
||||
..next
|
||||
};
|
||||
if next != current {
|
||||
rules::update(data, &next).await?;
|
||||
}
|
||||
response.updated.append(id, None);
|
||||
}
|
||||
|
||||
for id in request.unwrap_destroy().into_valid() {
|
||||
let Some(current) = (match rule_id(id) {
|
||||
Some(rule_id) => rules::get(data, rule_id).await?,
|
||||
None => None,
|
||||
}) else {
|
||||
response.not_destroyed.append(id, SetError::not_found());
|
||||
continue;
|
||||
};
|
||||
if !can_see(access_token, current.kind) {
|
||||
response.not_destroyed.append(id, SetError::not_found());
|
||||
continue;
|
||||
}
|
||||
if !can_change(access_token, current.kind) {
|
||||
response.not_destroyed.append(id, forbidden(current.kind));
|
||||
continue;
|
||||
}
|
||||
rules::delete(data, current.id).await?;
|
||||
response.destroyed.push(id);
|
||||
}
|
||||
|
||||
Ok(response)
|
||||
}
|
||||
@@ -10,6 +10,11 @@
|
||||
pub mod access;
|
||||
pub mod account_lock;
|
||||
pub mod legal_hold;
|
||||
pub mod mail_rule;
|
||||
pub mod security_acceptance;
|
||||
pub mod journal;
|
||||
pub mod held_message;
|
||||
pub mod dlp_settings;
|
||||
pub mod hold_export;
|
||||
pub mod hold_export_api;
|
||||
pub mod audit;
|
||||
@@ -17,6 +22,8 @@ pub mod audit_log;
|
||||
pub mod ai_limits;
|
||||
pub mod log_settings;
|
||||
pub mod data_inventory;
|
||||
pub mod directory_test;
|
||||
pub mod webhook_test;
|
||||
pub mod explanation;
|
||||
pub mod protocol_policy;
|
||||
pub mod tenant_protocol_policy;
|
||||
|
||||
@@ -0,0 +1,257 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! `inbuxa:SecurityAcceptance` (security to-do list spec, SS-23 to SS-26):
|
||||
//! the security to-do items an administrator accepted, with why, so every
|
||||
//! administrator sees the same accepted risks. Created and destroyed, never
|
||||
//! updated (the request gate refuses an update). Seeing them needs what the
|
||||
//! security page needs; changing them needs `sysSecurityAccept`. Every
|
||||
//! check is server-wide, so nobody in a tenant reaches them. The request
|
||||
//! layer records every change in the audit log (SS-26).
|
||||
|
||||
use common::{Server, auth::AccessToken};
|
||||
use inbuxa_features::security::acceptance::{self, Acceptance, Created};
|
||||
use jmap_proto::{
|
||||
error::set::SetError,
|
||||
method::{
|
||||
get::{GetRequest, GetResponse},
|
||||
set::{SetRequest, SetResponse},
|
||||
},
|
||||
object::inbuxa_security_acceptance::{
|
||||
SecurityAcceptance, SecurityAcceptanceProperty as P, SecurityAcceptanceValue,
|
||||
},
|
||||
request::IntoValid,
|
||||
types::date::UTCDate,
|
||||
};
|
||||
use jmap_tools::{Key, Map, Property, Value};
|
||||
use std::borrow::Cow;
|
||||
use store::write::now;
|
||||
use types::id::Id;
|
||||
|
||||
type RValue = Value<'static, P, SecurityAcceptanceValue>;
|
||||
|
||||
const ALL: &[P] = &[
|
||||
P::Id,
|
||||
P::Check,
|
||||
P::Subject,
|
||||
P::AcceptedValue,
|
||||
P::Note,
|
||||
P::AcceptedBy,
|
||||
P::AcceptedAt,
|
||||
];
|
||||
|
||||
/// Properties the server sets; a client that sends them is refused.
|
||||
const SERVER_SET: &[P] = &[P::Id, P::AcceptedBy, P::AcceptedAt];
|
||||
|
||||
fn server_level(access_token: &AccessToken) -> trc::Result<()> {
|
||||
if access_token.tenant_id().is_some() {
|
||||
Err(trc::JmapEvent::Forbidden
|
||||
.into_err()
|
||||
.details("Security checks are the server's."))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn json_to_value(json: serde_json::Value) -> RValue {
|
||||
match json {
|
||||
serde_json::Value::Null => Value::Null,
|
||||
serde_json::Value::Bool(b) => Value::Bool(b),
|
||||
serde_json::Value::Number(n) => {
|
||||
if let Some(n) = n.as_u64() {
|
||||
Value::Number(n.into())
|
||||
} else if let Some(n) = n.as_i64() {
|
||||
Value::Number(n.into())
|
||||
} else {
|
||||
Value::Number(n.as_f64().unwrap_or_default().into())
|
||||
}
|
||||
}
|
||||
serde_json::Value::String(s) => Value::Str(Cow::Owned(s)),
|
||||
serde_json::Value::Array(items) => {
|
||||
Value::Array(items.into_iter().map(json_to_value).collect())
|
||||
}
|
||||
serde_json::Value::Object(map) => {
|
||||
let mut out = Map::with_capacity(map.len());
|
||||
for (key, value) in map {
|
||||
out.insert_unchecked(Key::Owned(key), json_to_value(value));
|
||||
}
|
||||
Value::Object(out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn to_value(acceptance: &Acceptance, properties: &[P]) -> RValue {
|
||||
let mut out = Map::with_capacity(properties.len());
|
||||
for property in properties {
|
||||
let value = match property {
|
||||
P::Id => Value::Element(SecurityAcceptanceValue::Id(Id::from(acceptance.id))),
|
||||
P::Check => Value::Str(acceptance.check.clone().into()),
|
||||
P::Subject => Value::Str(acceptance.subject.clone().into()),
|
||||
P::AcceptedValue => json_to_value(acceptance.accepted_value.clone()),
|
||||
P::Note => Value::Str(acceptance.note.clone().into()),
|
||||
P::AcceptedBy => Value::Str(acceptance.accepted_by.clone().into()),
|
||||
P::AcceptedAt => Value::Str(
|
||||
UTCDate::from_timestamp(acceptance.accepted_at as i64)
|
||||
.to_string()
|
||||
.into(),
|
||||
),
|
||||
};
|
||||
out.insert_unchecked(Key::Property(property.clone()), value);
|
||||
}
|
||||
Value::Object(out)
|
||||
}
|
||||
|
||||
/// An acceptance as sent, checked whole.
|
||||
fn parse(value: Value<'_, P, SecurityAcceptanceValue>) -> Result<Acceptance, SetError<P>> {
|
||||
let mut map = serde_json::Map::new();
|
||||
for (key, value) in value.into_expanded_object() {
|
||||
match &key {
|
||||
Key::Property(p) if SERVER_SET.contains(p) => {
|
||||
return Err(SetError::invalid_properties()
|
||||
.with_property(p.clone())
|
||||
.with_description("The server sets this."));
|
||||
}
|
||||
Key::Property(p) => {
|
||||
map.insert(p.to_cow().into_owned(), value.into());
|
||||
}
|
||||
_ => {
|
||||
return Err(SetError::invalid_properties().with_property(key.clone().into_owned()));
|
||||
}
|
||||
}
|
||||
}
|
||||
let acceptance: Acceptance =
|
||||
serde_json::from_value(serde_json::Value::Object(map)).map_err(|err| {
|
||||
SetError::invalid_properties()
|
||||
.with_description(format!("Not a valid acceptance: {err}"))
|
||||
})?;
|
||||
acceptance.validate().map_err(|invalid| {
|
||||
let property = invalid.property.parse::<P>().unwrap_or(P::Note);
|
||||
SetError::invalid_properties()
|
||||
.with_property(property)
|
||||
.with_description(invalid.reason)
|
||||
})?;
|
||||
Ok(Acceptance {
|
||||
note: acceptance.note.trim().to_string(),
|
||||
..acceptance
|
||||
})
|
||||
}
|
||||
|
||||
fn acceptance_id(id: Id) -> Option<u32> {
|
||||
u32::try_from(id.id()).ok()
|
||||
}
|
||||
|
||||
/// `inbuxa:SecurityAcceptance/get`: every acceptance, oldest first.
|
||||
pub async fn get(
|
||||
server: &Server,
|
||||
access_token: &AccessToken,
|
||||
mut request: GetRequest<SecurityAcceptance>,
|
||||
) -> trc::Result<GetResponse<SecurityAcceptance>> {
|
||||
server_level(access_token)?;
|
||||
let properties = request.unwrap_properties(ALL);
|
||||
let (ids, not_found) = request.unwrap_ids(server.core.jmap.get_max_objects)?;
|
||||
let mut response = GetResponse {
|
||||
account_id: request.account_id.into(),
|
||||
state: None,
|
||||
list: Vec::new(),
|
||||
not_found,
|
||||
};
|
||||
let all = acceptance::all(server.store()).await?;
|
||||
match ids {
|
||||
None => {
|
||||
response.list = all.iter().map(|a| to_value(a, &properties)).collect();
|
||||
}
|
||||
Some(ids) => {
|
||||
for id in ids {
|
||||
match acceptance_id(id).and_then(|id| all.iter().find(|a| a.id == id)) {
|
||||
Some(a) => response.list.push(to_value(a, &properties)),
|
||||
None => response.push_not_found(id),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
/// `inbuxa:SecurityAcceptance/set`: accept an item, or remove an acceptance.
|
||||
pub async fn set(
|
||||
server: &Server,
|
||||
access_token: &AccessToken,
|
||||
mut request: SetRequest<'_, SecurityAcceptance>,
|
||||
) -> trc::Result<SetResponse<SecurityAcceptance>> {
|
||||
server_level(access_token)?;
|
||||
let mut response = SetResponse::from_request(&request, server.core.jmap.set_max_objects)?;
|
||||
let data = server.store();
|
||||
let actor = server.audit_actor(access_token).await;
|
||||
|
||||
for (client_id, value) in request.unwrap_create() {
|
||||
let parsed = match parse(value) {
|
||||
Ok(parsed) => parsed,
|
||||
Err(error) => {
|
||||
response.not_created.append(client_id, error);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let accepted = Acceptance {
|
||||
accepted_by: actor.name.clone(),
|
||||
accepted_at: now(),
|
||||
..parsed
|
||||
};
|
||||
match acceptance::create(data, &accepted).await? {
|
||||
Created::Id(id) => {
|
||||
let mut out = Map::with_capacity(3);
|
||||
out.insert_unchecked(
|
||||
Key::Property(P::Id),
|
||||
Value::Element(SecurityAcceptanceValue::Id(Id::from(id))),
|
||||
);
|
||||
out.insert_unchecked(
|
||||
Key::Property(P::AcceptedBy),
|
||||
Value::Str(accepted.accepted_by.clone().into()),
|
||||
);
|
||||
out.insert_unchecked(
|
||||
Key::Property(P::AcceptedAt),
|
||||
Value::Str(
|
||||
UTCDate::from_timestamp(accepted.accepted_at as i64)
|
||||
.to_string()
|
||||
.into(),
|
||||
),
|
||||
);
|
||||
response.created.insert(client_id, Value::Object(out));
|
||||
}
|
||||
Created::Full => {
|
||||
response.not_created.append(
|
||||
client_id,
|
||||
SetError::over_quota().with_description(format!(
|
||||
"There are already {} acceptances. Remove some first.",
|
||||
acceptance::MAX_ACCEPTANCES
|
||||
)),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (id, _) in request.unwrap_update().into_valid() {
|
||||
response.not_updated.append(
|
||||
id,
|
||||
SetError::forbidden().with_description("An acceptance is replaced, not edited."),
|
||||
);
|
||||
}
|
||||
|
||||
for id in request.unwrap_destroy().into_valid() {
|
||||
let found = match acceptance_id(id) {
|
||||
Some(acceptance_id) => acceptance::get(data, acceptance_id).await?,
|
||||
None => None,
|
||||
};
|
||||
match found {
|
||||
Some(found) => {
|
||||
acceptance::delete(data, found.id).await?;
|
||||
response.destroyed.push(id);
|
||||
}
|
||||
None => response.not_destroyed.append(id, SetError::not_found()),
|
||||
}
|
||||
}
|
||||
|
||||
Ok(response)
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! `POST /api/webhook/test`: send one sample event to a saved webhook
|
||||
//! (settings-reorg, Webhooks "Send test").
|
||||
//!
|
||||
//! ```json
|
||||
//! {"webhookId": "b"}
|
||||
//! ```
|
||||
//!
|
||||
//! The answer is `{"sent": true, "status": 200, "ms": 84}` when the receiver
|
||||
//! answered 2xx, `{"sent": false, "status": 403, …}` when it answered
|
||||
//! otherwise, and `{"sent": false, "error": "…"}` when nothing came back. The
|
||||
//! webhook is used as saved, even when it's off, so it can be tried before
|
||||
//! it's switched on. The request goes where the saved webhook already sends,
|
||||
//! so this gives nobody a reach they didn't have.
|
||||
//!
|
||||
//! For server-level administrators who may change webhooks.
|
||||
|
||||
use common::{Server, auth::AccessToken};
|
||||
use registry::schema::{enums::Permission, structs::WebHook};
|
||||
use serde_json::{Value, json};
|
||||
use std::{str::FromStr, time::Instant};
|
||||
use types::id::Id;
|
||||
|
||||
pub fn assert_allowed(access_token: &AccessToken) -> trc::Result<()> {
|
||||
if access_token.tenant_id().is_some() {
|
||||
return Err(trc::JmapEvent::Forbidden
|
||||
.into_err()
|
||||
.details("Webhook tests are for server-level administrators."));
|
||||
}
|
||||
access_token.enforce_permission(Permission::SysWebHookUpdate)
|
||||
}
|
||||
|
||||
pub async fn test(server: &Server, body: &Value) -> trc::Result<Value> {
|
||||
let webhook_id = body
|
||||
.get("webhookId")
|
||||
.and_then(Value::as_str)
|
||||
.and_then(|id| Id::from_str(id).ok())
|
||||
.ok_or_else(|| {
|
||||
trc::ResourceEvent::BadParameters
|
||||
.into_err()
|
||||
.details("Expected {\"webhookId\": …}")
|
||||
})?;
|
||||
let Some(hook) = server.registry().object::<WebHook>(webhook_id).await? else {
|
||||
return Ok(json!({ "sent": false, "error": "There's no such webhook. Save it first." }));
|
||||
};
|
||||
|
||||
let started = Instant::now();
|
||||
Ok(match common::telemetry::webhooks::send_test(&hook).await {
|
||||
Ok(status) => json!({
|
||||
"sent": (200..300).contains(&status),
|
||||
"status": status,
|
||||
"ms": started.elapsed().as_millis() as u64,
|
||||
}),
|
||||
Err(error) => json!({ "sent": false, "error": error }),
|
||||
})
|
||||
}
|
||||
@@ -49,6 +49,12 @@ use trc::AddContext;
|
||||
use types::{blob::BlobId, blob_hash::BlobHash, id::Id};
|
||||
use utils::map::vec_map::VecMap;
|
||||
|
||||
/// inbuxa: held mail is the review queue's to decide.
|
||||
fn held_refusal() -> SetError<Property> {
|
||||
SetError::forbidden()
|
||||
.with_description("This message is held for review: release or reject it under Compliance, Held mail.")
|
||||
}
|
||||
|
||||
pub(crate) async fn queued_message_set(
|
||||
mut set: RegistrySetResponse<'_>,
|
||||
) -> trc::Result<RegistrySetResponse<'_>> {
|
||||
@@ -66,6 +72,12 @@ pub(crate) async fn queued_message_set(
|
||||
let mut refresh_queue = false;
|
||||
'outer: for (id, value) in set.update.drain(..) {
|
||||
let queue_id = id.id();
|
||||
// inbuxa: held mail is released or rejected by review, not here
|
||||
// (dlp-and-mail-flow-rules spec, §2.6)
|
||||
if inbuxa_features::mailflow::held::is_held(set.server.store(), queue_id).await? {
|
||||
set.response.not_updated.append(id, held_refusal());
|
||||
continue;
|
||||
}
|
||||
let Some(archive) = set.server.read_message_archive(queue_id).await? else {
|
||||
set.response.not_updated.append(id, SetError::not_found());
|
||||
continue;
|
||||
@@ -238,6 +250,11 @@ pub(crate) async fn queued_message_set(
|
||||
|
||||
// Process destroy operations
|
||||
for id in set.destroy.drain(..) {
|
||||
// inbuxa: §2.6, as above
|
||||
if inbuxa_features::mailflow::held::is_held(set.server.store(), id.id()).await? {
|
||||
set.response.not_destroyed.append(id, held_refusal());
|
||||
continue;
|
||||
}
|
||||
let Some(message) = set.server.read_message(id.id(), QueueName::default()).await else {
|
||||
set.response.not_destroyed.append(id, SetError::not_found());
|
||||
continue;
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use common::{
|
||||
@@ -16,7 +18,7 @@ use email::{
|
||||
submission::{Address, Delivered, DeliveryStatus, EmailSubmission, UndoStatus},
|
||||
};
|
||||
use jmap_proto::{
|
||||
error::set::{SetError, SetErrorType},
|
||||
error::set::{DlpRule, SetError, SetErrorType},
|
||||
method::set::{SetRequest, SetResponse},
|
||||
object::email_submission::{self, EmailSubmissionProperty, EmailSubmissionValue},
|
||||
references::resolve::ResolveCreatedReference,
|
||||
@@ -89,6 +91,13 @@ impl EmailSubmissionSet for Server {
|
||||
);
|
||||
|
||||
let send_at = submission.send_at;
|
||||
// inbuxa: DLP (§2.6): the sender learns it's held
|
||||
let held = match submission.queue_id {
|
||||
Some(queue_id) => {
|
||||
inbuxa_features::mailflow::held::is_held(self.store(), queue_id).await?
|
||||
}
|
||||
None => false,
|
||||
};
|
||||
let undo_status = match submission.undo_status {
|
||||
UndoStatus::Pending => email_submission::UndoStatus::Pending,
|
||||
UndoStatus::Final => email_submission::UndoStatus::Final,
|
||||
@@ -126,7 +135,8 @@ impl EmailSubmissionSet for Server {
|
||||
.with_key_value(
|
||||
EmailSubmissionProperty::UndoStatus,
|
||||
Value::Element(EmailSubmissionValue::UndoStatus(undo_status)),
|
||||
),
|
||||
)
|
||||
.with_key_value(EmailSubmissionProperty::DlpHeld, Value::Bool(held)),
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -212,6 +222,17 @@ impl EmailSubmissionSet for Server {
|
||||
}
|
||||
|
||||
match undo_status {
|
||||
// inbuxa: held for review: the review decides, not an unsend
|
||||
// (dlp-and-mail-flow-rules spec, §2.6)
|
||||
Some(email_submission::UndoStatus::Canceled)
|
||||
if inbuxa_features::mailflow::held::is_held(self.store(), queue_id).await? =>
|
||||
{
|
||||
response.not_updated.append(
|
||||
id,
|
||||
SetError::new(SetErrorType::CannotUnsend)
|
||||
.with_description("The message is held for review and can't be unsent."),
|
||||
);
|
||||
}
|
||||
Some(email_submission::UndoStatus::Canceled) => {
|
||||
if let Some(queue_message) =
|
||||
self.read_message(queue_id, QueueName::default()).await
|
||||
@@ -379,6 +400,8 @@ impl EmailSubmissionSet for Server {
|
||||
};
|
||||
let mut mail_from: Option<MailFrom<Cow<'_, str>>> = None;
|
||||
let mut rcpt_to: Vec<RcptTo<Cow<'_, str>>> = Vec::new();
|
||||
// inbuxa: DLP (dlp-and-mail-flow-rules spec, §2.5)
|
||||
let mut dlp_override: Option<String> = None;
|
||||
|
||||
for (property, mut value) in object.into_expanded_object() {
|
||||
if let Err(err) = response.resolve_self_references(&mut value, 0, false) {
|
||||
@@ -493,6 +516,25 @@ impl EmailSubmissionSet for Server {
|
||||
(Key::Property(EmailSubmissionProperty::UndoStatus), Value::Element(_)) => {
|
||||
continue;
|
||||
}
|
||||
// inbuxa: the sender's reason to send despite a DLP warning
|
||||
(Key::Property(EmailSubmissionProperty::DlpOverride), Value::Object(value)) => {
|
||||
let reason = value
|
||||
.iter()
|
||||
.find(|(key, _)| key.to_string() == "reason")
|
||||
.and_then(|(_, value)| value.as_str().map(|r| r.trim().to_string()))
|
||||
.filter(|r| !r.is_empty());
|
||||
match reason {
|
||||
Some(reason) => dlp_override = Some(reason.chars().take(500).collect()),
|
||||
None => {
|
||||
return Ok(Err(SetError::invalid_properties()
|
||||
.with_property(EmailSubmissionProperty::DlpOverride)
|
||||
.with_description("An override needs a reason.")));
|
||||
}
|
||||
}
|
||||
}
|
||||
(Key::Property(EmailSubmissionProperty::DlpOverride), Value::Null) => {
|
||||
continue;
|
||||
}
|
||||
_ => {
|
||||
return Ok(Err(SetError::invalid_properties()
|
||||
.with_property(property.into_owned())
|
||||
@@ -700,6 +742,7 @@ impl EmailSubmissionSet for Server {
|
||||
0,
|
||||
),
|
||||
);
|
||||
session.data.dlp_override = dlp_override;
|
||||
|
||||
// Spawn SMTP session to avoid overflowing the stack
|
||||
let handle = tokio::spawn(async move {
|
||||
@@ -730,6 +773,27 @@ impl EmailSubmissionSet for Server {
|
||||
let response = session.queue_message().await;
|
||||
if let smtp::core::State::Accepted(queue_id) = session.state {
|
||||
Ok((responses, Some(queue_id)))
|
||||
} else if let Some(refusal) = session.data.dlp_refusal.take() {
|
||||
// inbuxa: DLP (§2.5): which rules, and what they say
|
||||
let description = refusal
|
||||
.rules
|
||||
.iter()
|
||||
.map(|(_, notice)| notice.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
Err(SetError::new(if refusal.blocked {
|
||||
SetErrorType::DlpBlocked
|
||||
} else {
|
||||
SetErrorType::DlpWarning
|
||||
})
|
||||
.with_description(description)
|
||||
.with_dlp_rules(
|
||||
refusal
|
||||
.rules
|
||||
.into_iter()
|
||||
.map(|(name, notice)| DlpRule { name, notice })
|
||||
.collect(),
|
||||
))
|
||||
} else {
|
||||
Err(
|
||||
SetError::new(SetErrorType::ForbiddenToSend).with_description(format!(
|
||||
|
||||
@@ -1748,6 +1748,20 @@ pub enum Permission {
|
||||
SysLegalHoldExport = 672,
|
||||
// inbuxa: personal-data catalog, the data inventory and compliance overview
|
||||
SysComplianceGet = 673,
|
||||
// inbuxa: DLP and mail flow rules
|
||||
SysMailRuleGet = 674,
|
||||
SysMailRuleUpdate = 675,
|
||||
SysDlpPolicyGet = 676,
|
||||
SysDlpPolicyUpdate = 677,
|
||||
SysDlpReviewGet = 678,
|
||||
SysDlpReviewUpdate = 679,
|
||||
// inbuxa: journaling
|
||||
SysJournalGet = 680,
|
||||
SysJournalUpdate = 681,
|
||||
SysJournalSearch = 682,
|
||||
SysJournalExport = 683,
|
||||
// inbuxa: the security to-do list, accepting an item
|
||||
SysSecurityAccept = 684,
|
||||
SysAccountGet = 219,
|
||||
SysAccountCreate = 220,
|
||||
SysAccountUpdate = 221,
|
||||
|
||||
@@ -7091,6 +7091,17 @@ impl EnumImpl for Permission {
|
||||
b"sysLegalHoldUpdate" => Permission::SysLegalHoldUpdate,
|
||||
b"sysLegalHoldExport" => Permission::SysLegalHoldExport,
|
||||
b"sysComplianceGet" => Permission::SysComplianceGet,
|
||||
b"sysMailRuleGet" => Permission::SysMailRuleGet,
|
||||
b"sysMailRuleUpdate" => Permission::SysMailRuleUpdate,
|
||||
b"sysDlpPolicyGet" => Permission::SysDlpPolicyGet,
|
||||
b"sysDlpPolicyUpdate" => Permission::SysDlpPolicyUpdate,
|
||||
b"sysDlpReviewGet" => Permission::SysDlpReviewGet,
|
||||
b"sysDlpReviewUpdate" => Permission::SysDlpReviewUpdate,
|
||||
b"sysJournalGet" => Permission::SysJournalGet,
|
||||
b"sysJournalUpdate" => Permission::SysJournalUpdate,
|
||||
b"sysJournalSearch" => Permission::SysJournalSearch,
|
||||
b"sysJournalExport" => Permission::SysJournalExport,
|
||||
b"sysSecurityAccept" => Permission::SysSecurityAccept,
|
||||
b"sysAccountGet" => Permission::SysAccountGet,
|
||||
b"sysAccountCreate" => Permission::SysAccountCreate,
|
||||
b"sysAccountUpdate" => Permission::SysAccountUpdate,
|
||||
@@ -7781,6 +7792,17 @@ impl EnumImpl for Permission {
|
||||
Permission::SysLegalHoldUpdate => "sysLegalHoldUpdate",
|
||||
Permission::SysLegalHoldExport => "sysLegalHoldExport",
|
||||
Permission::SysComplianceGet => "sysComplianceGet",
|
||||
Permission::SysMailRuleGet => "sysMailRuleGet",
|
||||
Permission::SysMailRuleUpdate => "sysMailRuleUpdate",
|
||||
Permission::SysDlpPolicyGet => "sysDlpPolicyGet",
|
||||
Permission::SysDlpPolicyUpdate => "sysDlpPolicyUpdate",
|
||||
Permission::SysDlpReviewGet => "sysDlpReviewGet",
|
||||
Permission::SysDlpReviewUpdate => "sysDlpReviewUpdate",
|
||||
Permission::SysJournalGet => "sysJournalGet",
|
||||
Permission::SysJournalUpdate => "sysJournalUpdate",
|
||||
Permission::SysJournalSearch => "sysJournalSearch",
|
||||
Permission::SysJournalExport => "sysJournalExport",
|
||||
Permission::SysSecurityAccept => "sysSecurityAccept",
|
||||
Permission::SysAccountGet => "sysAccountGet",
|
||||
Permission::SysAccountCreate => "sysAccountCreate",
|
||||
Permission::SysAccountUpdate => "sysAccountUpdate",
|
||||
@@ -8464,6 +8486,17 @@ impl EnumImpl for Permission {
|
||||
671 => Some(Permission::SysLegalHoldUpdate),
|
||||
672 => Some(Permission::SysLegalHoldExport),
|
||||
673 => Some(Permission::SysComplianceGet),
|
||||
674 => Some(Permission::SysMailRuleGet),
|
||||
675 => Some(Permission::SysMailRuleUpdate),
|
||||
676 => Some(Permission::SysDlpPolicyGet),
|
||||
677 => Some(Permission::SysDlpPolicyUpdate),
|
||||
678 => Some(Permission::SysDlpReviewGet),
|
||||
679 => Some(Permission::SysDlpReviewUpdate),
|
||||
680 => Some(Permission::SysJournalGet),
|
||||
681 => Some(Permission::SysJournalUpdate),
|
||||
682 => Some(Permission::SysJournalSearch),
|
||||
683 => Some(Permission::SysJournalExport),
|
||||
684 => Some(Permission::SysSecurityAccept),
|
||||
219 => Some(Permission::SysAccountGet),
|
||||
220 => Some(Permission::SysAccountCreate),
|
||||
221 => Some(Permission::SysAccountUpdate),
|
||||
@@ -8908,7 +8941,7 @@ impl EnumImpl for Permission {
|
||||
}
|
||||
}
|
||||
|
||||
const COUNT: usize = 674;
|
||||
const COUNT: usize = 685;
|
||||
}
|
||||
|
||||
impl serde::Serialize for Permission {
|
||||
|
||||
@@ -286,6 +286,17 @@ async fn store_maintenance(
|
||||
trc::error!(err.details("Failed to purge expired IP bans"));
|
||||
}
|
||||
|
||||
// inbuxa: DLP, §2.6: mail nobody reviewed in time goes back
|
||||
if let Err(err) = smtp::queue::held::expire(server).await {
|
||||
trc::error!(err.details("Failed to return unreviewed held mail"));
|
||||
}
|
||||
|
||||
// inbuxa: journaling, JR-13: entries past their retention go,
|
||||
// except those a legal hold keeps
|
||||
if let Err(err) = purge_journal(server).await {
|
||||
trc::error!(err.details("Failed to purge journal entries"));
|
||||
}
|
||||
|
||||
// inbuxa: AU-7: audit records past their retention go; a
|
||||
// failure leaves them for the next run
|
||||
if let Err(err) = server.audit_purge().await {
|
||||
@@ -404,6 +415,54 @@ async fn store_maintenance(
|
||||
Ok(TaskResult::Success(vec![]))
|
||||
}
|
||||
|
||||
/// inbuxa: journaling, JR-13: removes journal entries past their
|
||||
/// retention, keeping any whose sender or recipients a legal hold covers
|
||||
/// (deleted accounts a hold keeps included), and records how many went.
|
||||
async fn purge_journal(server: &Server) -> trc::Result<()> {
|
||||
use inbuxa_features::audit::{Action, Actor, Outcome, Record, Target};
|
||||
let mut held = server.held_accounts().await?;
|
||||
if !held.is_empty() {
|
||||
for (account_id, kept) in
|
||||
inbuxa_features::undelete::data::kept_accounts(server.store()).await?
|
||||
{
|
||||
if server.is_kept_held(account_id, &kept).await? {
|
||||
held.insert(account_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
let at = store::write::now();
|
||||
let purged = inbuxa_features::journal::entries::purge(server.store(), at, |entry| {
|
||||
entry.accounts.iter().any(|account| held.contains(account))
|
||||
})
|
||||
.await?;
|
||||
if purged.removed > 0 || purged.kept_for_hold > 0 {
|
||||
server
|
||||
.audit_note(Record {
|
||||
at: at * 1000,
|
||||
actor: Actor::system("Journal"),
|
||||
via: None,
|
||||
remote_ip: None,
|
||||
action: Action::Destroy,
|
||||
target: Target {
|
||||
kind: "inbuxa:JournalEntry".into(),
|
||||
id: None,
|
||||
name: None,
|
||||
account_id: None,
|
||||
tenant_id: None,
|
||||
},
|
||||
changes: vec![],
|
||||
details: Some(format!(
|
||||
"{} past their retention removed; {} kept for a legal hold",
|
||||
purged.removed, purged.kept_for_hold
|
||||
)),
|
||||
reason: None,
|
||||
outcome: Outcome::success(),
|
||||
})
|
||||
.await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn account_maintenance(
|
||||
server: &Server,
|
||||
task: &TaskAccountMaintenance,
|
||||
|
||||
@@ -46,6 +46,8 @@ enum Event {
|
||||
StoreMetrics,
|
||||
// inbuxa: MON-25: alert evaluation
|
||||
EvaluateAlerts,
|
||||
// inbuxa: settings-reorg: probe the other nodes' ports
|
||||
ProbePeerPorts,
|
||||
}
|
||||
|
||||
/// When the next metric-history tick is due (MON-4), read from the registry
|
||||
@@ -95,6 +97,11 @@ pub fn spawn_task_scheduler(inner: Arc<Inner>) {
|
||||
Instant::now() + server.registry().refresh_node_id_interval(),
|
||||
Event::RenewNodeIdLease,
|
||||
);
|
||||
// inbuxa: first round a minute after start, once the others have a lease
|
||||
queue.schedule(
|
||||
Instant::now() + Duration::from_secs(60),
|
||||
Event::ProbePeerPorts,
|
||||
);
|
||||
}
|
||||
|
||||
// Spam classifier training
|
||||
@@ -229,6 +236,19 @@ pub fn spawn_task_scheduler(inner: Arc<Inner>) {
|
||||
}
|
||||
});
|
||||
}
|
||||
Event::ProbePeerPorts => {
|
||||
queue.schedule(
|
||||
Instant::now() + common::reachability::PROBE_INTERVAL,
|
||||
Event::ProbePeerPorts,
|
||||
);
|
||||
|
||||
let server = server.clone();
|
||||
tokio::spawn(async move {
|
||||
if let Err(err) = common::reachability::probe_peers(&server).await {
|
||||
trc::error!(err.details("Failed to probe the other nodes' ports"));
|
||||
}
|
||||
});
|
||||
}
|
||||
Event::OtelMetrics => {
|
||||
if let Some(otel) = &server.core.metrics.otel {
|
||||
queue.schedule(Instant::now() + otel.interval, Event::OtelMetrics);
|
||||
@@ -476,6 +496,7 @@ impl Event {
|
||||
Event::RenewNodeIdLease => "renewNodeIdLease",
|
||||
Event::StoreMetrics => "storeMetrics",
|
||||
Event::EvaluateAlerts => "evaluateAlerts",
|
||||
Event::ProbePeerPorts => "probePeerPorts",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use crate::{inbound::auth::SaslToken, queue::QueueId};
|
||||
@@ -92,6 +94,22 @@ pub struct SessionData {
|
||||
pub spf_ehlo: Option<SpfOutput>,
|
||||
pub spf_mail_from: Option<SpfOutput>,
|
||||
pub dnsbl_error: Option<Vec<u8>>,
|
||||
|
||||
// inbuxa: DLP (dlp-and-mail-flow-rules spec, §2.5): the reason a JMAP
|
||||
// sender gave to send despite a warning, and why DATA refused a
|
||||
// message, for the submission to report
|
||||
pub dlp_override: Option<String>,
|
||||
pub dlp_refusal: Option<DlpRefusal>,
|
||||
// inbuxa: a mail flow rule's route for this message
|
||||
pub mailflow_queue: Option<String>,
|
||||
}
|
||||
|
||||
/// inbuxa: a DATA refusal by DLP rules: blocked, or a warning the sender
|
||||
/// may override, with each rule's name and notice.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DlpRefusal {
|
||||
pub blocked: bool,
|
||||
pub rules: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
@@ -168,6 +186,9 @@ impl SessionData {
|
||||
spf_ehlo: None,
|
||||
spf_mail_from: None,
|
||||
dnsbl_error: None,
|
||||
dlp_override: None,
|
||||
dlp_refusal: None,
|
||||
mailflow_queue: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -291,6 +312,9 @@ impl SessionData {
|
||||
spf_ehlo: None,
|
||||
spf_mail_from: None,
|
||||
dnsbl_error: None,
|
||||
dlp_override: None,
|
||||
dlp_refusal: None,
|
||||
mailflow_queue: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -738,6 +738,50 @@ impl<T: SessionStream> Session<T> {
|
||||
}
|
||||
}
|
||||
|
||||
// inbuxa: DLP (dlp-and-mail-flow-rules spec, §2.1): after the system
|
||||
// script, before headers and signing
|
||||
let mut held_draft = None;
|
||||
let (message, envelope) = match self
|
||||
.check_mail_rules(edited_message.as_deref().unwrap_or(raw_message.as_slice()))
|
||||
.await
|
||||
{
|
||||
super::mailflow::Checked::Accept => (None, Vec::new()),
|
||||
super::mailflow::Checked::Changed { message, envelope } => (message, envelope),
|
||||
// §2.6: queued, but not due for a century; a reviewer releases it
|
||||
super::mailflow::Checked::Hold { draft, message, envelope } => {
|
||||
self.data.future_release = inbuxa_features::mailflow::held::HOLD_SECONDS;
|
||||
held_draft = Some(draft);
|
||||
(message, envelope)
|
||||
}
|
||||
super::mailflow::Checked::Refuse(reply, refusal) => {
|
||||
self.data.dlp_refusal = refusal;
|
||||
return reply.into();
|
||||
}
|
||||
};
|
||||
if let Some(message) = message {
|
||||
edited_message = Some(message);
|
||||
}
|
||||
for change in envelope {
|
||||
match change {
|
||||
super::mailflow::EnvelopeChange::AddRecipient(address) => {
|
||||
if !self
|
||||
.data
|
||||
.rcpt_to
|
||||
.iter()
|
||||
.any(|r| r.address_lcase.eq_ignore_ascii_case(&address))
|
||||
{
|
||||
self.data.rcpt_to.push(SessionAddress::new(address));
|
||||
}
|
||||
}
|
||||
super::mailflow::EnvelopeChange::Redirect(addresses) => {
|
||||
self.data.rcpt_to = addresses.into_iter().map(SessionAddress::new).collect();
|
||||
}
|
||||
super::mailflow::EnvelopeChange::Route(queue) => {
|
||||
self.data.mailflow_queue = Some(queue);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build message
|
||||
let mail_from = self.data.mail_from.clone().unwrap();
|
||||
let rcpt_to = std::mem::take(&mut self.data.rcpt_to);
|
||||
@@ -817,6 +861,19 @@ impl<T: SessionStream> Session<T> {
|
||||
.server
|
||||
.eval_signers(&ac.dkim.sign, self, self.data.session_id)
|
||||
.await;
|
||||
// inbuxa: §2.6, who the held message is from and to
|
||||
let held_envelope = held_draft.as_ref().map(|_| {
|
||||
(
|
||||
message.message.return_path.to_string(),
|
||||
message
|
||||
.message
|
||||
.recipients
|
||||
.iter()
|
||||
.map(|r| r.address.to_string())
|
||||
.collect::<Vec<_>>(),
|
||||
message.message.size,
|
||||
)
|
||||
});
|
||||
if message
|
||||
.queue(
|
||||
QueueParams::new(raw_message, self.data.session_id, &self.server)
|
||||
@@ -831,9 +888,17 @@ impl<T: SessionStream> Session<T> {
|
||||
{
|
||||
self.state = State::Accepted(queue_id);
|
||||
self.data.messages_sent += 1;
|
||||
if let (Some(draft), Some((sender, recipients, size))) = (held_draft, held_envelope)
|
||||
{
|
||||
self.record_held(queue_id, draft, sender, recipients, size).await;
|
||||
format!("250 2.0.0 Held for review, id {queue_id:x}.\r\n")
|
||||
.into_bytes()
|
||||
.into()
|
||||
} else {
|
||||
format!("250 2.0.0 Message queued with id {queue_id:x}.\r\n")
|
||||
.into_bytes()
|
||||
.into()
|
||||
}
|
||||
} else {
|
||||
(b"451 4.3.5 Unable to accept message at this time.\r\n"[..]).into()
|
||||
}
|
||||
@@ -903,8 +968,10 @@ impl<T: SessionStream> Session<T> {
|
||||
};
|
||||
|
||||
// Resolve queue
|
||||
let queue = self.server.get_queue_or_default(
|
||||
&self
|
||||
// inbuxa: a mail flow rule's route comes before the strategy
|
||||
let queue_name = match &self.data.mailflow_queue {
|
||||
Some(queue) => queue.clone(),
|
||||
None => self
|
||||
.server
|
||||
.eval_if::<String, _>(
|
||||
&self.server.core.smtp.queue.queue,
|
||||
@@ -913,8 +980,10 @@ impl<T: SessionStream> Session<T> {
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|| "default".to_string()),
|
||||
self.data.session_id,
|
||||
);
|
||||
};
|
||||
let queue = self
|
||||
.server
|
||||
.get_queue_or_default(&queue_name, self.data.session_id);
|
||||
|
||||
// Set expiration and notification times
|
||||
let num_intervals = std::cmp::max(queue.notify.len(), 1);
|
||||
|
||||
@@ -0,0 +1,651 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! inbuxa: DLP at DATA (dlp-and-mail-flow-rules spec, §2.1, §2.4–§2.7).
|
||||
//!
|
||||
//! Runs after the DATA system script and before headers and DKIM signing,
|
||||
//! on mail an authenticated sender submits over SMTP or JMAP. The rules and
|
||||
//! the detectors are `inbuxa_features::mailflow`; this is the glue: build
|
||||
//! what they look at from the message, apply the decision, record it.
|
||||
|
||||
use crate::core::{DlpRefusal, Session};
|
||||
use common::network::SessionStream;
|
||||
use inbuxa_features::{
|
||||
audit::{Action, Actor, Outcome, Record, Target},
|
||||
mailflow::{
|
||||
cache,
|
||||
engine::{
|
||||
Attachment, Content, Decision, Envelope, Outcome as RulesOutcome, Recipient, RuleRef,
|
||||
},
|
||||
extract::{self, Extracted, Limits},
|
||||
held::{self, Held, HeldRule, KEEP_DAYS},
|
||||
rewrite,
|
||||
rules::{Action as RuleAction, Kind},
|
||||
},
|
||||
};
|
||||
use mail_parser::{Message, MessageParser, MimeHeaders, PartType};
|
||||
use std::{borrow::Cow, time::SystemTime};
|
||||
|
||||
/// How much text one message is read for; past it, the rest counts as
|
||||
/// "can't be inspected" (§2.3).
|
||||
const INSPECTION_LIMIT: usize = 10 * 1024 * 1024;
|
||||
|
||||
/// What the check decided.
|
||||
pub enum Checked {
|
||||
/// Go on, with the message unchanged.
|
||||
Accept,
|
||||
/// Go on, with a changed message (the override tag taken out, a
|
||||
/// disclaimer, headers, the subject) and envelope.
|
||||
Changed {
|
||||
message: Option<Vec<u8>>,
|
||||
envelope: Vec<EnvelopeChange>,
|
||||
},
|
||||
/// Refuse, with this SMTP reply, and for a JMAP submission, why.
|
||||
Refuse(Vec<u8>, Option<DlpRefusal>),
|
||||
/// Queue it held for review (§2.6), with any transport changes.
|
||||
Hold {
|
||||
draft: HeldDraft,
|
||||
message: Option<Vec<u8>>,
|
||||
envelope: Vec<EnvelopeChange>,
|
||||
},
|
||||
}
|
||||
|
||||
/// What the review record will say, once the message has a queue id.
|
||||
pub struct HeldDraft {
|
||||
pub subject: String,
|
||||
pub rules: Vec<HeldRule>,
|
||||
pub counts: Vec<(String, usize)>,
|
||||
pub notify_sender: bool,
|
||||
}
|
||||
|
||||
/// What a transport rule changes about where a message goes.
|
||||
pub enum EnvelopeChange {
|
||||
AddRecipient(String),
|
||||
Redirect(Vec<String>),
|
||||
Route(String),
|
||||
}
|
||||
|
||||
/// `[override: reason]` at the start of a subject: the reason, and the
|
||||
/// subject without it.
|
||||
pub fn override_tag(subject: &str) -> Option<(String, String)> {
|
||||
let trimmed = subject.trim_start();
|
||||
let head = trimmed.get(..10)?;
|
||||
if !head.eq_ignore_ascii_case("[override:") {
|
||||
return None;
|
||||
}
|
||||
let close = trimmed.find(']')?;
|
||||
let reason = trimmed[10..close].trim();
|
||||
if reason.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some((
|
||||
reason.chars().take(500).collect(),
|
||||
trimmed[close + 1..].trim_start().to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
/// One line of an SMTP reply: no line breaks, a sane length.
|
||||
fn reply_text(text: &str) -> String {
|
||||
text.split_whitespace()
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
.chars()
|
||||
.take(400)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn notices(rules: &[RuleRef]) -> String {
|
||||
let mut seen = Vec::new();
|
||||
for rule in rules {
|
||||
let notice = reply_text(&rule.notice);
|
||||
if !seen.contains(¬ice) {
|
||||
seen.push(notice);
|
||||
}
|
||||
}
|
||||
seen.join(" ")
|
||||
}
|
||||
|
||||
fn refusal(blocked: bool, rules: &[RuleRef]) -> DlpRefusal {
|
||||
DlpRefusal {
|
||||
blocked,
|
||||
rules: rules
|
||||
.iter()
|
||||
.map(|r| (r.name.clone(), r.notice.clone()))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
/// A message and the messages attached to it, one level down.
|
||||
fn collect<'x>(message: &'x Message<'x>, content: &mut Content<'x>, budget: &mut usize, depth: u8) {
|
||||
let add = |text: Cow<'x, str>, content: &mut Content<'x>, budget: &mut usize| {
|
||||
if *budget == 0 {
|
||||
content.truncated = true;
|
||||
return;
|
||||
}
|
||||
if text.len() > *budget {
|
||||
let mut cut = *budget;
|
||||
while !text.is_char_boundary(cut) {
|
||||
cut -= 1;
|
||||
}
|
||||
content.bodies.push(Cow::Owned(text[..cut].to_string()));
|
||||
content.truncated = true;
|
||||
*budget = 0;
|
||||
} else {
|
||||
*budget -= text.len();
|
||||
content.bodies.push(text);
|
||||
}
|
||||
};
|
||||
// The text version of each body (an HTML-only one converted), not both
|
||||
// versions of the same alternative, so words aren't counted twice
|
||||
for part in message.text_bodies() {
|
||||
match &part.body {
|
||||
PartType::Text(text) => add(Cow::Borrowed(text.as_ref()), content, budget),
|
||||
PartType::Html(html) => add(
|
||||
Cow::Owned(mail_parser::decoders::html::html_to_text(html)),
|
||||
content,
|
||||
budget,
|
||||
),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
for part in message.attachments() {
|
||||
if let (Some(inner), true) = (part.message(), depth == 0) {
|
||||
if let Some(subject) = inner.subject() {
|
||||
add(Cow::Borrowed(subject), content, budget);
|
||||
}
|
||||
collect(inner, content, budget, depth + 1);
|
||||
continue;
|
||||
}
|
||||
let content_type = part
|
||||
.content_type()
|
||||
.map(|ct| match ct.subtype() {
|
||||
Some(sub) => format!("{}/{}", ct.ctype(), sub),
|
||||
None => ct.ctype().to_string(),
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let bytes = part.contents();
|
||||
let mut extracted = extract::extract(
|
||||
&content_type,
|
||||
part.attachment_name(),
|
||||
bytes,
|
||||
&Limits::default(),
|
||||
);
|
||||
if let Extracted::Text(text) = &extracted {
|
||||
if text.len() > *budget {
|
||||
extracted = Extracted::NotInspectable(extract::Why::TooLarge);
|
||||
} else {
|
||||
*budget -= text.len();
|
||||
}
|
||||
}
|
||||
content.attachments.push(Attachment {
|
||||
name: part.attachment_name(),
|
||||
content_type: content_type.into(),
|
||||
size: bytes.len() as u64,
|
||||
extracted,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: SessionStream> Session<T> {
|
||||
/// DLP on an outgoing message (§2.4). `message` is what the DATA stage
|
||||
/// has so far (the script's replacement, if it made one).
|
||||
pub async fn check_mail_rules(&self, message: &[u8]) -> Checked {
|
||||
// Outgoing: an authenticated sender. DLP rules check outgoing mail
|
||||
// only (settled); transport rules may check either
|
||||
let sender = self
|
||||
.data
|
||||
.authenticated_as
|
||||
.as_ref()
|
||||
.map(|s| (s.account_id, s.account.clone()));
|
||||
let outgoing = sender.is_some();
|
||||
let rules = match cache::compiled(self.server.store()).await {
|
||||
Ok(rules) => rules,
|
||||
Err(err) => {
|
||||
trc::error!(
|
||||
err.span_id(self.data.session_id)
|
||||
.caused_by(trc::location!())
|
||||
.details("Failed to load mail rules")
|
||||
);
|
||||
// Fail closed: a message nobody could check doesn't leave
|
||||
return Checked::Refuse(
|
||||
b"451 4.3.0 This message couldn't be checked against the server's rules. Try again later.\r\n"
|
||||
.to_vec(),
|
||||
None,
|
||||
);
|
||||
}
|
||||
};
|
||||
if !rules.applies_to(outgoing) {
|
||||
return Checked::Accept;
|
||||
}
|
||||
|
||||
let parsed = MessageParser::new().parse(message);
|
||||
let subject = parsed
|
||||
.as_ref()
|
||||
.and_then(|m| m.subject())
|
||||
.unwrap_or_default();
|
||||
let jmap_override = self.data.dlp_override.clone();
|
||||
// Only a sender of ours can override
|
||||
let tag = if outgoing {
|
||||
override_tag(subject)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let checked_subject = tag.as_ref().map_or(subject, |(_, rest)| rest.as_str());
|
||||
let held_subject = checked_subject.to_string();
|
||||
|
||||
let mut content = Content {
|
||||
subject: checked_subject,
|
||||
size: message.len() as u64,
|
||||
..Default::default()
|
||||
};
|
||||
let mut budget = INSPECTION_LIMIT;
|
||||
match &parsed {
|
||||
Some(parsed) => {
|
||||
content.headers = parsed
|
||||
.headers()
|
||||
.iter()
|
||||
.filter_map(|h| h.value.as_text().map(|v| (h.name.as_str(), v)))
|
||||
.collect();
|
||||
collect(parsed, &mut content, &mut budget, 0);
|
||||
}
|
||||
// Nothing a rule could read: say so, rather than pass it
|
||||
None => content.truncated = true,
|
||||
}
|
||||
let mut recipient_groups = Vec::with_capacity(self.data.rcpt_to.len());
|
||||
for rcpt in &self.data.rcpt_to {
|
||||
let local = self
|
||||
.server
|
||||
.domain(&rcpt.domain)
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
.is_some();
|
||||
let groups = if local {
|
||||
match self
|
||||
.server
|
||||
.account_id_from_email(&rcpt.address_lcase, false)
|
||||
.await
|
||||
{
|
||||
Ok(Some(id)) => self
|
||||
.server
|
||||
.account(id)
|
||||
.await
|
||||
.map(|a| a.id_member_of.to_vec())
|
||||
.unwrap_or_default(),
|
||||
_ => Vec::new(),
|
||||
}
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
recipient_groups.push((local, groups));
|
||||
}
|
||||
let sender_address = self
|
||||
.data
|
||||
.mail_from
|
||||
.as_ref()
|
||||
.map(|m| m.address_lcase.clone())
|
||||
.unwrap_or_default();
|
||||
let envelope = Envelope {
|
||||
outgoing,
|
||||
sender: &sender_address,
|
||||
sender_groups: sender
|
||||
.as_ref()
|
||||
.map_or(&[][..], |(_, a)| &a.id_member_of[..]),
|
||||
sender_tenant: sender.as_ref().and_then(|(_, a)| a.id_tenant),
|
||||
recipients: self
|
||||
.data
|
||||
.rcpt_to
|
||||
.iter()
|
||||
.zip(&recipient_groups)
|
||||
.map(|(rcpt, (local, groups))| Recipient {
|
||||
address: &rcpt.address_lcase,
|
||||
local: *local,
|
||||
groups,
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
|
||||
let outcome = rules.evaluate(&envelope, &content);
|
||||
let override_reason =
|
||||
jmap_override.or_else(|| tag.as_ref().map(|(reason, _)| reason.clone()));
|
||||
let decision = outcome.decision(override_reason.is_some());
|
||||
let mut domains: Vec<&str> = self
|
||||
.data
|
||||
.rcpt_to
|
||||
.iter()
|
||||
.map(|r| r.domain.as_str())
|
||||
.collect();
|
||||
domains.sort_unstable();
|
||||
domains.dedup();
|
||||
let domains = domains.join(", ");
|
||||
drop(envelope);
|
||||
|
||||
if let Some((account_id, account)) = &sender {
|
||||
self.record_dlp(
|
||||
*account_id,
|
||||
account,
|
||||
&outcome,
|
||||
&decision,
|
||||
override_reason.as_deref(),
|
||||
&domains,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
let hold = match decision {
|
||||
Decision::Block(rules) => {
|
||||
let refusal = refusal(true, &rules);
|
||||
return Checked::Refuse(
|
||||
format!("550 5.7.1 {}\r\n", notices(&rules)).into_bytes(),
|
||||
Some(refusal),
|
||||
);
|
||||
}
|
||||
Decision::Warn(rules) => {
|
||||
return Checked::Refuse(
|
||||
format!(
|
||||
"550 5.7.1 {} To send anyway, start the subject with [override: your reason]\r\n",
|
||||
notices(&rules)
|
||||
)
|
||||
.into_bytes(),
|
||||
Some(refusal(false, &rules)),
|
||||
);
|
||||
}
|
||||
// Accepted and queued, but not sent until a reviewer says so
|
||||
// (§2.6); the transport rules still apply, so what's released
|
||||
// is what would have gone out
|
||||
Decision::Hold {
|
||||
rules,
|
||||
notify_sender,
|
||||
} => Some(HeldDraft {
|
||||
subject: held_subject,
|
||||
rules: rules
|
||||
.iter()
|
||||
.map(|r| HeldRule {
|
||||
name: r.name.clone(),
|
||||
notice: r.notice.clone(),
|
||||
})
|
||||
.collect(),
|
||||
counts: outcome
|
||||
.matched
|
||||
.iter()
|
||||
.filter(|m| m.kind == Kind::Dlp)
|
||||
.flat_map(|m| m.counts.iter().cloned())
|
||||
.collect(),
|
||||
notify_sender,
|
||||
}),
|
||||
Decision::Pass => None,
|
||||
};
|
||||
{
|
||||
{
|
||||
// The tag was an instruction to the server, not part of the
|
||||
// subject: it doesn't go out
|
||||
let mut current: Option<Vec<u8>> =
|
||||
tag.map(|(_, rest)| rewrite::set_subject(message, &rest));
|
||||
let mut changes = Vec::new();
|
||||
for matched in outcome.matched.iter().filter(|m| m.kind == Kind::Transport) {
|
||||
for action in &matched.actions {
|
||||
let now = current.as_deref().unwrap_or(message);
|
||||
let next = match action {
|
||||
RuleAction::AddDisclaimer {
|
||||
text,
|
||||
html,
|
||||
position,
|
||||
} => rewrite::add_disclaimer(now, text, html.as_deref(), *position),
|
||||
RuleAction::AddHeader { name, value } => {
|
||||
Some(rewrite::add_header(now, name, value))
|
||||
}
|
||||
RuleAction::RemoveHeader { name } => rewrite::remove_header(now, name),
|
||||
RuleAction::PrefixSubject { text } => {
|
||||
rewrite::prefix_subject(now, text)
|
||||
}
|
||||
RuleAction::AddRecipient { address } => {
|
||||
changes.push(EnvelopeChange::AddRecipient(address.clone()));
|
||||
None
|
||||
}
|
||||
RuleAction::Redirect { addresses } => {
|
||||
changes.push(EnvelopeChange::Redirect(addresses.clone()));
|
||||
None
|
||||
}
|
||||
RuleAction::Route { queue } => {
|
||||
changes.push(EnvelopeChange::Route(queue.clone()));
|
||||
None
|
||||
}
|
||||
RuleAction::Refuse { text } => {
|
||||
self.record_transport(&sender, &matched.name, "refused", &domains)
|
||||
.await;
|
||||
return Checked::Refuse(
|
||||
format!("550 5.7.1 {}\r\n", reply_text(text)).into_bytes(),
|
||||
None,
|
||||
);
|
||||
}
|
||||
RuleAction::Block { .. }
|
||||
| RuleAction::Warn { .. }
|
||||
| RuleAction::Hold { .. } => None,
|
||||
};
|
||||
if next.is_some() {
|
||||
current = next;
|
||||
}
|
||||
}
|
||||
// Where mail goes is recorded; wording and headers aren't,
|
||||
// or a banner rule would write a record for every message
|
||||
let routed: Vec<String> = matched
|
||||
.actions
|
||||
.iter()
|
||||
.filter_map(|a| match a {
|
||||
RuleAction::AddRecipient { address } => {
|
||||
Some(format!("copied to {address}"))
|
||||
}
|
||||
RuleAction::Redirect { addresses } => {
|
||||
Some(format!("redirected to {}", addresses.join(", ")))
|
||||
}
|
||||
RuleAction::Route { queue } => Some(format!("routed through {queue}")),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
if !routed.is_empty() {
|
||||
self.record_transport(&sender, &matched.name, &routed.join(", "), &domains)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
match hold {
|
||||
Some(draft) => Checked::Hold {
|
||||
draft,
|
||||
message: current,
|
||||
envelope: changes,
|
||||
},
|
||||
None if current.is_none() && changes.is_empty() => Checked::Accept,
|
||||
None => Checked::Changed {
|
||||
message: current,
|
||||
envelope: changes,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Writes the review record for a message just queued held (§2.6),
|
||||
/// and tells the sender when the rule asks. A failure to write it is
|
||||
/// logged: the message stays held, never sent unreviewed.
|
||||
pub async fn record_held(
|
||||
&self,
|
||||
queue_id: u64,
|
||||
draft: HeldDraft,
|
||||
sender: String,
|
||||
recipients: Vec<String>,
|
||||
size: u64,
|
||||
) {
|
||||
let at = store::write::now();
|
||||
let keep_days = held::settings(self.server.store())
|
||||
.await
|
||||
.map_or(KEEP_DAYS, |s| s.keep_held_days);
|
||||
let account = self.data.authenticated_as.as_ref();
|
||||
let record = Held {
|
||||
queue_id,
|
||||
sender,
|
||||
account_id: account.map(|a| a.account_id),
|
||||
tenant_id: account.and_then(|a| a.account.id_tenant),
|
||||
recipients,
|
||||
subject: draft.subject,
|
||||
size,
|
||||
rules: draft.rules,
|
||||
counts: draft.counts,
|
||||
held_at: at,
|
||||
expires_at: at + keep_days * 86_400,
|
||||
keep_days,
|
||||
};
|
||||
if let Err(err) = held::create(self.server.store(), &record).await {
|
||||
trc::error!(
|
||||
err.span_id(self.data.session_id)
|
||||
.caused_by(trc::location!())
|
||||
.details("Failed to write the review record of a held message")
|
||||
);
|
||||
return;
|
||||
}
|
||||
if draft.notify_sender {
|
||||
crate::queue::held::notify_held(&self.server, &record).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// A transport rule that refused a message or changed where it goes
|
||||
/// (§2.7): who sent it (or the server, for incoming mail), the rule,
|
||||
/// what it did.
|
||||
async fn record_transport(
|
||||
&self,
|
||||
sender: &Option<(u32, std::sync::Arc<common::auth::AccountCache>)>,
|
||||
rule: &str,
|
||||
what: &str,
|
||||
domains: &str,
|
||||
) {
|
||||
let (actor, account_id, tenant_id) = match sender {
|
||||
Some((id, account)) => (
|
||||
Actor::account(*id, account.name.to_string(), account.id_tenant),
|
||||
Some(*id),
|
||||
account.id_tenant,
|
||||
),
|
||||
None => (Actor::system("mail-flow"), None, None),
|
||||
};
|
||||
let at = SystemTime::now()
|
||||
.duration_since(SystemTime::UNIX_EPOCH)
|
||||
.map_or(0, |d| d.as_millis() as u64);
|
||||
self.server
|
||||
.audit_note(Record {
|
||||
at,
|
||||
actor,
|
||||
via: None,
|
||||
remote_ip: Some(self.data.remote_ip),
|
||||
action: Action::Create,
|
||||
target: Target {
|
||||
kind: "message".into(),
|
||||
id: None,
|
||||
name: None,
|
||||
account_id,
|
||||
tenant_id,
|
||||
},
|
||||
changes: vec![],
|
||||
details: Some(format!("Mail flow rule \"{rule}\" {what}, to {domains}")),
|
||||
reason: None,
|
||||
outcome: if what == "refused" {
|
||||
Outcome::refused("forbidden", None)
|
||||
} else {
|
||||
Outcome::success()
|
||||
},
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
/// One audit record per message a DLP rule matched (§2.7): who sent it,
|
||||
/// where to, which rules and each detector's count, what happened, and
|
||||
/// an override's reason. Never the matched text.
|
||||
async fn record_dlp(
|
||||
&self,
|
||||
account_id: u32,
|
||||
account: &common::auth::AccountCache,
|
||||
outcome: &RulesOutcome,
|
||||
decision: &Decision,
|
||||
override_reason: Option<&str>,
|
||||
domains: &str,
|
||||
) {
|
||||
let dlp: Vec<_> = outcome
|
||||
.matched
|
||||
.iter()
|
||||
.filter(|m| m.kind == Kind::Dlp)
|
||||
.collect();
|
||||
if dlp.is_empty() {
|
||||
return;
|
||||
}
|
||||
let rules = dlp
|
||||
.iter()
|
||||
.map(|m| {
|
||||
let counts = m
|
||||
.counts
|
||||
.iter()
|
||||
.map(|(id, n)| format!("{id} {n}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
if counts.is_empty() {
|
||||
format!("\"{}\"", m.name)
|
||||
} else {
|
||||
format!("\"{}\" ({counts})", m.name)
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("; ");
|
||||
let (what, outcome, reason) = match decision {
|
||||
Decision::Hold { .. } => ("held for review", Outcome::success(), None),
|
||||
Decision::Block(_) => ("blocked", Outcome::refused("inbuxa:dlpBlocked", None), None),
|
||||
Decision::Warn(_) => ("warned", Outcome::refused("inbuxa:dlpWarning", None), None),
|
||||
Decision::Pass => (
|
||||
"sent after a warning",
|
||||
Outcome::success(),
|
||||
override_reason.map(str::to_string),
|
||||
),
|
||||
};
|
||||
let at = SystemTime::now()
|
||||
.duration_since(SystemTime::UNIX_EPOCH)
|
||||
.map_or(0, |d| d.as_millis() as u64);
|
||||
self.server
|
||||
.audit_note(Record {
|
||||
at,
|
||||
actor: Actor::account(account_id, account.name.to_string(), account.id_tenant),
|
||||
via: None,
|
||||
remote_ip: Some(self.data.remote_ip),
|
||||
action: Action::Create,
|
||||
target: Target {
|
||||
kind: "message".into(),
|
||||
id: None,
|
||||
name: None,
|
||||
account_id: Some(account_id),
|
||||
tenant_id: account.id_tenant,
|
||||
},
|
||||
changes: vec![],
|
||||
details: Some(format!("DLP {what}, to {domains}: {rules}")),
|
||||
reason,
|
||||
outcome,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::override_tag;
|
||||
|
||||
#[test]
|
||||
fn override_tags() {
|
||||
assert_eq!(
|
||||
override_tag("[override: client asked for it] Card details"),
|
||||
Some(("client asked for it".into(), "Card details".into()))
|
||||
);
|
||||
assert_eq!(
|
||||
override_tag(" [OVERRIDE:yes]x"),
|
||||
Some(("yes".into(), "x".into()))
|
||||
);
|
||||
assert_eq!(override_tag("[override: ] x"), None);
|
||||
assert_eq!(override_tag("Re: [override: no] x"), None);
|
||||
assert_eq!(override_tag("[override: unclosed"), None);
|
||||
assert_eq!(override_tag(""), None);
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,8 @@
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use mail_auth::{DkimResult, DmarcResult, IprevResult, SpfResult, dmarc::Policy};
|
||||
@@ -13,6 +15,7 @@ pub mod dkim;
|
||||
pub mod ehlo;
|
||||
pub mod hooks;
|
||||
pub mod mail;
|
||||
pub mod mailflow; // inbuxa: DLP and mail flow rules
|
||||
pub mod milter;
|
||||
pub mod rcpt;
|
||||
pub mod session;
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! inbuxa: mail held for review (dlp-and-mail-flow-rules spec, §2.6):
|
||||
//! releasing it, rejecting it, rejecting what nobody reviewed in time, and
|
||||
//! the notices the sender gets.
|
||||
//!
|
||||
//! A held message sits in the queue with its release [`HOLD_SECONDS`] off.
|
||||
//! Releasing it undoes exactly that: each recipient due now, its next
|
||||
//! notice as far from now as it was from its retry, its lifetime counted
|
||||
//! from the release.
|
||||
|
||||
use crate::{
|
||||
queue::{Message, MessageWrapper, Status, spool::SmtpSpool},
|
||||
reporting::send::MtaReportSend,
|
||||
};
|
||||
use common::{
|
||||
Server,
|
||||
config::smtp::queue::{QueueExpiry, QueueName},
|
||||
ipc::QueueEvent,
|
||||
};
|
||||
use inbuxa_features::{
|
||||
audit::{Action, Actor, Outcome, Record, Target},
|
||||
mailflow::held::{self, HOLD_SECONDS, Held},
|
||||
};
|
||||
use mail_builder::{
|
||||
MessageBuilder,
|
||||
headers::{HeaderType, address::Address},
|
||||
};
|
||||
use store::{ahash::AHashSet, write::now};
|
||||
|
||||
/// Puts a held message back on its way. False when it's no longer queued.
|
||||
pub async fn release(server: &Server, queue_id: u64) -> trc::Result<bool> {
|
||||
let Some(archive) = server.read_message_archive(queue_id).await? else {
|
||||
held::delete(server.store(), queue_id).await?;
|
||||
return Ok(false);
|
||||
};
|
||||
let mut message: Message = archive.to_unarchived::<Message>()?.deserialize()?;
|
||||
let prev_events = message.next_events();
|
||||
let at = now();
|
||||
let mut modified = AHashSet::new();
|
||||
for (idx, rcpt) in message.recipients.iter_mut().enumerate() {
|
||||
if !matches!(rcpt.status, Status::Scheduled | Status::TemporaryFailure(_)) {
|
||||
continue;
|
||||
}
|
||||
let notify_gap = rcpt.notify.due.saturating_sub(rcpt.retry.due);
|
||||
rcpt.retry.due = at;
|
||||
rcpt.notify.due = at + notify_gap;
|
||||
if let QueueExpiry::Ttl(ttl) = rcpt.expires {
|
||||
rcpt.expires = QueueExpiry::Ttl(
|
||||
ttl.saturating_sub(HOLD_SECONDS) + at.saturating_sub(message.created),
|
||||
);
|
||||
}
|
||||
modified.insert(idx);
|
||||
}
|
||||
let saved = MessageWrapper::new(message, queue_id, QueueName::default())
|
||||
.save_registry_changes(server, prev_events, modified)
|
||||
.await;
|
||||
held::delete(server.store(), queue_id).await?;
|
||||
let _ = server.inner.ipc.queue_tx.send(QueueEvent::Refresh).await;
|
||||
Ok(saved)
|
||||
}
|
||||
|
||||
/// Takes a held message out of the queue and tells its sender, with the
|
||||
/// reviewer's note if there is one. False when it's no longer queued.
|
||||
pub async fn reject(server: &Server, record: &Held, note: Option<&str>) -> trc::Result<bool> {
|
||||
let removed = match server
|
||||
.read_message(record.queue_id, QueueName::default())
|
||||
.await
|
||||
{
|
||||
Some(message) => message.remove(server, None).await,
|
||||
None => false,
|
||||
};
|
||||
held::delete(server.store(), record.queue_id).await?;
|
||||
let mut text = format!(
|
||||
"Your message \"{}\" to {} was held for review under this server's rules, and wasn't sent.\r\n",
|
||||
record.subject,
|
||||
record.recipients.join(", ")
|
||||
);
|
||||
match note {
|
||||
Some(note) => text.push_str(&format!("\r\nThe reviewer's note: {note}\r\n")),
|
||||
None => text.push_str(&format!(
|
||||
"\r\nNobody reviewed it within {} days, so it was returned.\r\n",
|
||||
record.keep_days
|
||||
)),
|
||||
}
|
||||
notify(
|
||||
server,
|
||||
record,
|
||||
&format!("Not sent: {}", record.subject),
|
||||
text,
|
||||
)
|
||||
.await;
|
||||
let _ = server.inner.ipc.queue_tx.send(QueueEvent::Refresh).await;
|
||||
Ok(removed)
|
||||
}
|
||||
|
||||
/// Tells the sender their message is held (when the rule asks).
|
||||
pub async fn notify_held(server: &Server, record: &Held) {
|
||||
let notices = record
|
||||
.rules
|
||||
.iter()
|
||||
.map(|r| r.notice.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
let text = format!(
|
||||
"Your message \"{}\" to {} is held for review under this server's rules: {notices}\r\n\r\n\
|
||||
It will be sent if a reviewer releases it, and returned otherwise within {} days.\r\n",
|
||||
record.subject,
|
||||
record.recipients.join(", "),
|
||||
record.keep_days,
|
||||
);
|
||||
notify(
|
||||
server,
|
||||
record,
|
||||
&format!("Held for review: {}", record.subject),
|
||||
text,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn notify(server: &Server, record: &Held, subject: &str, text: String) {
|
||||
let domain = record
|
||||
.sender
|
||||
.rsplit_once('@')
|
||||
.map_or("localhost", |(_, d)| d);
|
||||
let from = format!("postmaster@{domain}");
|
||||
let message = MessageBuilder::new()
|
||||
.from(Address::new_address(Some("Mail review"), from.clone()))
|
||||
.to(Address::new_address(None::<String>, record.sender.clone()))
|
||||
.subject(subject)
|
||||
.header("Auto-Submitted", HeaderType::Text("auto-replied".into()))
|
||||
.text_body(text)
|
||||
.write_to_vec()
|
||||
.unwrap_or_default();
|
||||
server
|
||||
.send_autogenerated(from, [record.sender.as_str()].into_iter(), message, None, 0)
|
||||
.await;
|
||||
}
|
||||
|
||||
/// Rejects every held message nobody reviewed in time (§2.6), each
|
||||
/// recorded as the server's doing. Returns how many.
|
||||
pub async fn expire(server: &Server) -> trc::Result<usize> {
|
||||
let at = now();
|
||||
let mut count = 0;
|
||||
for record in held::all(server.store()).await? {
|
||||
if !record.is_expired(at) {
|
||||
continue;
|
||||
}
|
||||
reject(server, &record, None).await?;
|
||||
count += 1;
|
||||
server
|
||||
.audit_note(Record {
|
||||
at: at * 1000,
|
||||
actor: Actor::system("DLP"),
|
||||
via: None,
|
||||
remote_ip: None,
|
||||
action: Action::Destroy,
|
||||
target: Target {
|
||||
kind: "inbuxa:HeldMessage".into(),
|
||||
id: Some(record.queue_id.to_string()),
|
||||
name: Some(record.subject.clone()),
|
||||
account_id: record.account_id,
|
||||
tenant_id: record.tenant_id,
|
||||
},
|
||||
changes: vec![],
|
||||
details: Some(format!(
|
||||
"Rejected: nobody reviewed it within {} days; the sender was told",
|
||||
record.keep_days
|
||||
)),
|
||||
reason: None,
|
||||
outcome: Outcome::success(),
|
||||
})
|
||||
.await;
|
||||
}
|
||||
Ok(count)
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! inbuxa: journaling (journaling spec, JR-1 to JR-5, JR-11): the copy
|
||||
//! taken as a message is queued, after DLP and transport rules, so it has
|
||||
//! the envelope the message actually leaves or arrives with.
|
||||
|
||||
use crate::queue::{FROM_AUTHENTICATED, FROM_AUTOGENERATED, FROM_DSN, FROM_REPORT, Message};
|
||||
use common::Server;
|
||||
use inbuxa_features::{
|
||||
hold::Member,
|
||||
journal::{
|
||||
self, Direction,
|
||||
entries::{self, Entry},
|
||||
report::{self, Envelope, Recipient},
|
||||
},
|
||||
mailflow::held::HOLD_SECONDS,
|
||||
};
|
||||
use store::write::{BatchBuilder, BlobLink, BlobOp, now};
|
||||
use types::blob_hash::BlobHash;
|
||||
|
||||
/// Marks a journal report the server queued itself, so it's never
|
||||
/// journaled (JR-2). Free in the message flags (the MAIL parameters use
|
||||
/// the low bits, the sources bits 32 to 37).
|
||||
pub const FROM_JOURNAL: u64 = 1 << 48;
|
||||
|
||||
/// Journals `message`, whose queued bytes are `raw`, into every enabled
|
||||
/// journal that takes it. An error means it may not have been journaled,
|
||||
/// and the caller must not queue it.
|
||||
pub async fn capture(
|
||||
server: &Server,
|
||||
queue_id: u64,
|
||||
message: &Message,
|
||||
raw: &[u8],
|
||||
) -> trc::Result<()> {
|
||||
if message.flags & (FROM_JOURNAL | FROM_REPORT) != 0 {
|
||||
return Ok(());
|
||||
}
|
||||
let journals = journal::enabled(server.store()).await?;
|
||||
if journals.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Who's here on either side, and which way it goes
|
||||
let mut members: Vec<Member> = Vec::new();
|
||||
let mut sender_local = message.flags & FROM_AUTHENTICATED != 0
|
||||
|| (message.return_path.is_empty() && message.flags & (FROM_DSN | FROM_AUTOGENERATED) != 0);
|
||||
if !message.return_path.is_empty()
|
||||
&& let Some(id) = server
|
||||
.account_id_from_email(&message.return_path, false)
|
||||
.await?
|
||||
{
|
||||
sender_local = true;
|
||||
if let Some(member) = server.member_of(id).await {
|
||||
members.push(member);
|
||||
}
|
||||
}
|
||||
let (mut any_local, mut any_remote) = (false, false);
|
||||
for rcpt in &message.recipients {
|
||||
let address = rcpt.address.to_lowercase();
|
||||
let domain = address.rsplit_once('@').map_or("", |(_, d)| d);
|
||||
let local_domain = server.domain(domain).await.ok().flatten().is_some();
|
||||
match server.account_id_from_email(&address, false).await? {
|
||||
Some(id) => {
|
||||
any_local = true;
|
||||
if !members.iter().any(|m| m.account == id)
|
||||
&& let Some(member) = server.member_of(id).await
|
||||
{
|
||||
members.push(member);
|
||||
}
|
||||
}
|
||||
None if local_domain => any_local = true,
|
||||
None => any_remote = true,
|
||||
}
|
||||
}
|
||||
let direction = Direction::of(sender_local, any_remote, any_local);
|
||||
let taken: Vec<&journal::Journal> = journals
|
||||
.iter()
|
||||
.filter(|j| j.takes(direction, &members))
|
||||
.collect();
|
||||
if taken.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// DLP holds a message by putting its release a century off
|
||||
let at = now();
|
||||
let held = !message.recipients.is_empty()
|
||||
&& message
|
||||
.recipients
|
||||
.iter()
|
||||
.all(|rcpt| rcpt.retry.due >= at + HOLD_SECONDS / 2);
|
||||
let recipients: Vec<Recipient> = message
|
||||
.recipients
|
||||
.iter()
|
||||
.map(|rcpt| Recipient {
|
||||
address: rcpt.address.to_string(),
|
||||
orcpt: rcpt.orcpt.as_deref().map(Into::into),
|
||||
})
|
||||
.collect();
|
||||
let envelope = Envelope {
|
||||
sender: &message.return_path,
|
||||
authenticated: message.flags & FROM_AUTHENTICATED != 0,
|
||||
recipients: &recipients,
|
||||
queue_id,
|
||||
received: message.created,
|
||||
direction,
|
||||
held,
|
||||
};
|
||||
let host = server.core.network.server_name.as_str();
|
||||
let (bytes, fields) = report::build(&envelope, raw, &format!("postmaster@{host}"), host);
|
||||
|
||||
// The report's blob, reserved until the entry links it
|
||||
let hash = BlobHash::generate(&bytes);
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.set(
|
||||
BlobOp::Link {
|
||||
hash: hash.clone(),
|
||||
to: BlobLink::Temporary { until: at + 120 },
|
||||
},
|
||||
vec![],
|
||||
);
|
||||
server.store().write(batch.build_all()).await?;
|
||||
server
|
||||
.blob_store()
|
||||
.put_blob(hash.as_slice(), &bytes, server.core.email.compression)
|
||||
.await?;
|
||||
|
||||
let retention_days = taken
|
||||
.iter()
|
||||
.map(|j| j.retention_days)
|
||||
.max()
|
||||
.unwrap_or_default();
|
||||
let mut tenants: Vec<u32> = members.iter().filter_map(|m| m.tenant).collect();
|
||||
tenants.sort_unstable();
|
||||
tenants.dedup();
|
||||
let entry = Entry {
|
||||
queue_id,
|
||||
at,
|
||||
direction,
|
||||
sender: message.return_path.to_string(),
|
||||
authenticated: envelope.authenticated,
|
||||
recipients: recipients.iter().map(|r| r.address.clone()).collect(),
|
||||
subject: fields.subject,
|
||||
message_id: fields.message_id,
|
||||
accounts: members.iter().map(|m| m.account).collect(),
|
||||
tenants,
|
||||
journals: taken.iter().map(|j| j.id).collect(),
|
||||
held,
|
||||
blob: entries::hex(hash.as_slice()),
|
||||
size: bytes.len() as u64,
|
||||
sha256: entries::sha256(&bytes),
|
||||
expires_at: at + u64::from(retention_days) * 86_400,
|
||||
};
|
||||
entries::append(server.store(), server.core.network.node_id, &entry).await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -2,6 +2,8 @@
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use common::{
|
||||
@@ -21,6 +23,8 @@ use types::blob_hash::BlobHash;
|
||||
use utils::DomainPart;
|
||||
|
||||
pub mod dsn;
|
||||
pub mod held; // inbuxa: mail held for review
|
||||
pub mod journal; // inbuxa: journaling
|
||||
pub mod manager;
|
||||
pub mod quota;
|
||||
pub mod spool;
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use super::{
|
||||
@@ -453,6 +455,25 @@ impl MessageWrapper {
|
||||
return false;
|
||||
}
|
||||
|
||||
// inbuxa: journaling, JR-1: the copy is taken before the message is
|
||||
// queued; if it can't be, the message isn't queued either
|
||||
if let Err(err) = crate::queue::journal::capture(
|
||||
server,
|
||||
self.queue_id,
|
||||
&self.message,
|
||||
message.as_ref(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
trc::error!(
|
||||
err.details("Failed to journal a message.")
|
||||
.span_id(session_id)
|
||||
.caused_by(trc::location!())
|
||||
);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
trc::event!(
|
||||
Queue(event),
|
||||
SpanId = session_id,
|
||||
|
||||
@@ -81,7 +81,7 @@ fn legacy_setting(name: &str, is_set: impl Fn(&str) -> bool) -> Option<String> {
|
||||
#[macro_export]
|
||||
macro_rules! brand_version {
|
||||
() => {
|
||||
"2026.9.28.4"
|
||||
"2026.9.28.5"
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -362,6 +362,8 @@ is written.
|
||||
| 9 | Per-domain directories | A domain signs in against its own LDAP, SQL or OIDC directory | Added 2026-09-18. Signing in through an OIDC provider as the server's directory is already AGPL; only the per-domain choice is Enterprise. Built 2026-09-19 in `crates/common/src/auth` and `crates/directory`; status in `features/per-domain-directories.md`. |
|
||||
| — | Seat limits, license keys | Nothing: there's no license | Removed, not rebuilt. |
|
||||
|
||||
Not a rebuild: the **security to-do list** is INBUXA's own design (inbuxa-drafts `specs/security-score.md`). The console runs its checks; the server's part is `inbuxa:SecurityAcceptance`, the accepted items (`crates/jmap/src/inbuxa/security_acceptance.rs`), and the `sysSecurityAccept` permission.
|
||||
|
||||
## 5. The web front ends
|
||||
|
||||
**Which ihasmail.** Public ihasmail stays Stalwart-facing: its code, docs,
|
||||
|
||||
@@ -0,0 +1,443 @@
|
||||
# Feature spec: data loss prevention and mail flow rules
|
||||
|
||||
Status: **approved 2026-09-28**, with the answers under [Settled](#settled)
|
||||
and the detector catalog in §2.3. Phase 1 of DLP and the rule builder, specced together because they
|
||||
need the same conditions, the same place in the mail path and the same record
|
||||
of what matched. Not a rebuild of an upstream feature, so it has no line in
|
||||
SPEC.md §4's table.
|
||||
|
||||
## Provenance
|
||||
|
||||
Written for the record SPEC.md §3 rule 3 asks for. Sources, and nothing else:
|
||||
|
||||
| Source | License | Used for |
|
||||
|---|---|---|
|
||||
| This repository at `0502eb4` (2026-09-28): `crates/smtp/src/inbound/data.rs`, `crates/smtp/src/queue/`, `crates/jmap/src/submission/set.rs`, `crates/common/src/scripts/`, `vendor/sieve-rs`, `resources/schema/schema.json.gz` | AGPL-3.0-only | Where a check can run, what the queue stores, what a sender sees on a refusal |
|
||||
| inbuxa-admin at `b82904c` | AGPL-3.0-only | Where the pages go |
|
||||
| ihasmail-inbuxa (the webmail) at `290bc63` | AGPL-3.0-or-later | How a refused send reaches the person sending |
|
||||
| `inbuxa-drafts/queue/dlp.md`, `rule-builder.md` | Own | What John asked for and settled |
|
||||
| The personal-data catalog spec and the audit-hold-lock spec | Own | Roles, the audit log, legal holds, the catalog check |
|
||||
| RFC 5321, RFC 3463 (enhanced status codes), RFC 8620/8621 (JMAP) | Public | Refusal codes and the submission error shape |
|
||||
| The issuing authorities' published formats and check-digit rules for each identifier in §2.3 (ISO 13616, ISO/IEC 7812, ISO 7064, and each national scheme's own publication) | Public | The detector rules; each is implemented from its publication and tested against its published examples |
|
||||
|
||||
No Enterprise-only file or snippet was used, and no third-party DLP product
|
||||
was consulted for design: the detectors are public checksum and format rules
|
||||
(Luhn, ISO 13616 mod 97, the SSA's published SSN rules).
|
||||
|
||||
## What it is
|
||||
|
||||
1. **Data loss prevention (DLP).** Policies that look at mail as someone
|
||||
sends it, find what shouldn't leave (card numbers, bank accounts, national
|
||||
ID numbers, words and patterns an organization names, attachments of a
|
||||
kind), and then **block** it with a notice, **warn** and let the sender
|
||||
send anyway with a stated reason, or **hold** it until a reviewer releases
|
||||
or rejects it.
|
||||
2. **Mail flow rules.** The same engine, for ordinary transport rules an
|
||||
administrator writes in a form instead of in Sieve: disclaimers, banners,
|
||||
headers, copies, redirects, refusals.
|
||||
3. **One record of what matched**, in the audit log, and a **review queue**
|
||||
for held mail.
|
||||
|
||||
Settled before this spec (John, 2026-09-27 and 2026-09-28): DLP's first
|
||||
version checks **outgoing mail only**, content and attachments, as it's sent;
|
||||
its actions are block with a notice, warn with an override (audited), and hold
|
||||
for review. A record-only action was not chosen. The rule builder goes
|
||||
alongside DLP, one design; journaling comes after both.
|
||||
|
||||
**Out of scope** (later specs): inbound DLP, files and calendar sharing,
|
||||
scanning mail already stored, a machine-learning classifier (the local AI
|
||||
model could add one later; nothing here depends on it), mobile and ihasmail
|
||||
screens, journaling.
|
||||
|
||||
Nothing in code, docs, UI text or output claims the product meets a legal
|
||||
standard or prevents every leak. The pages say what a policy checks and what
|
||||
it did.
|
||||
|
||||
## 1. What exists today
|
||||
|
||||
Checked by reading the code at `0502eb4`:
|
||||
|
||||
| Need | Today |
|
||||
|---|---|
|
||||
| A place in the send path that sees every outgoing message | Yes. SMTP submission and webmail sends both reach `Session::queue_message` (`inbound/data.rs`); JMAP submission builds a local session and runs MAIL, RCPT and DATA (`jmap/src/submission/set.rs` ~L690–760). Nothing reaches the queue around it. |
|
||||
| Order at DATA | Authentication checks → spam filter → milters → MTA hooks → the DATA system Sieve script → headers, DKIM signing → queue. |
|
||||
| Rules without code | System Sieve (`x:SieveSystemScript`): one script per stage, chosen by an expression on `x:MtaStageData.script`. Hand-written only; the console has a text field. |
|
||||
| Refusing a message | A 5xx at DATA. The webmail gets `forbiddenToSend` with the text `Server rejected DATA: <reply>` and nothing structured. |
|
||||
| Holding a message | **Nowhere.** "Quarantine" in the code is only DMARC's disposition. The queue's recipient status is `Scheduled`, `Completed`, `TemporaryFailure`, `PermanentFailure`, archived with rkyv. |
|
||||
| Reading attachments | Text and HTML parts only. There's no PDF or Office text extraction: search indexes text parts and file names. |
|
||||
| Recording what happened | The audit log, with system actors, reasons and outcomes (AU-1…AU-12). |
|
||||
| Who is allowed | Roles with per-permission grants; server and tenant levels; the compliance roles from the catalog spec. |
|
||||
|
||||
## 2. Design
|
||||
|
||||
### 2.1 One engine, native, after the system script
|
||||
|
||||
Rules are evaluated by a new native engine at DATA, **after** the system Sieve
|
||||
script and before headers and DKIM signing. Mail flow rules and DLP policies
|
||||
are two views of the same rule list.
|
||||
|
||||
Why not generate Sieve: holding a message, a warning the sender can override,
|
||||
counted detectors with checksums, and a per-rule match record are all things
|
||||
Sieve doesn't have. Adding them means new extensions in `vendor/sieve-rs`,
|
||||
which widens the fork of a crate we'd otherwise take from upstream, and a
|
||||
generated script would have to share the single DATA script with whatever an
|
||||
administrator wrote by hand. A native engine leaves hand-written Sieve exactly
|
||||
as it is: it still runs, first, and the rules see its result.
|
||||
|
||||
The engine lives in `crates/features/src/mailflow/` (pure evaluation over a
|
||||
parsed message and an envelope, unit-testable), called from `data.rs` behind
|
||||
one `// inbuxa:` marked block.
|
||||
|
||||
### 2.2 Rules
|
||||
|
||||
A fork-owned JMAP object, `inbuxa:MailRule`, stored in inbuxa's own subspace
|
||||
like legal holds (not a registry object, so upstream schema imports never
|
||||
touch it):
|
||||
|
||||
| Property | |
|
||||
|---|---|
|
||||
| `name`, `description` | |
|
||||
| `kind` | `dlp` or `transport`: which page shows it and which permission edits it |
|
||||
| `enabled` | |
|
||||
| `priority` | Order; lower runs first |
|
||||
| `direction` | `outgoing` (authenticated senders), `incoming`, or `any`. **DLP rules are `outgoing` only** in this version. |
|
||||
| `conditions` | All must match (list below) |
|
||||
| `exceptions` | Any matching one skips the rule |
|
||||
| `actions` | What happens (list below) |
|
||||
| `stopProcessing` | Later rules don't run for this message |
|
||||
| `tenantId` | Always none in this version: rules are server-level (settled answer 3); a **Tenant** condition narrows a rule to tenants |
|
||||
| `createdBy`, `updatedAt` | |
|
||||
|
||||
Every create, update and delete is audited with its before and after, like
|
||||
any setting, and appears on the compliance Overview's **Changes that affect
|
||||
review**.
|
||||
|
||||
### 2.3 Conditions
|
||||
|
||||
Shared by both kinds:
|
||||
|
||||
| Condition | Matches when |
|
||||
|---|---|
|
||||
| Sender | the sender is one of the chosen accounts, or in a chosen group, domain or tenant |
|
||||
| Tenant | the sender is in one of the chosen tenants |
|
||||
| Recipient | any recipient is one of the chosen addresses, domains, groups |
|
||||
| **Recipient outside** | any recipient isn't at a domain this server hosts |
|
||||
| Subject or body contains | any of a list of words or phrases (whole words, case-insensitive) |
|
||||
| Subject or body matches | a regular expression (the `regex` crate: linear time, no backtracking) |
|
||||
| Header | a header exists, or its value contains or matches |
|
||||
| Attachment | its detected type, extension or name matches; its size is over a limit; there are more than N |
|
||||
| **Can't be inspected** | an attachment is encrypted or password-protected (ZIP, PDF, Office), or bigger than the inspection limit |
|
||||
| Message size | over a limit |
|
||||
|
||||
DLP adds **detectors**. Each counts what it finds, and a rule sets a minimum
|
||||
(for example "5 or more card numbers"). A detector is one of two strengths:
|
||||
|
||||
- **Checked**: the identifier has a published check digit or checksum, so a
|
||||
random number rarely passes. Found on its own.
|
||||
- **Needs a word**: the format is too common to trust alone (nine digits, a
|
||||
date). Counted only with a corroborating word nearby, within 50 characters
|
||||
either side, in the languages where the identifier is used ("passport",
|
||||
"Reisepass", "pasaporte"...).
|
||||
|
||||
A check that about one random number in ten passes (Luhn, mod 10, mod 11) is
|
||||
too weak for a bare run of digits: invoice and phone numbers would match. So
|
||||
a checked identifier that is only digits (SSN, SIN, NHS, TFN, Medicare…)
|
||||
counts alone in the written form it's issued in (`536-22-1234`,
|
||||
`130 692 544`, `943 476 5919`), and as bare digits only beside a word. ABA
|
||||
routing numbers and NPIs are never written with separators, so they always
|
||||
need a word. (Refinement made while building phase 2, 2026-09-28.)
|
||||
|
||||
The catalog (settled answer 6: the recognized, protected identifiers, not a
|
||||
chosen few). Each row is one table entry and one check function in
|
||||
`crates/features/src/mailflow/detectors/`:
|
||||
|
||||
| Region | Detector | Strength | Rule |
|
||||
|---|---|---|---|
|
||||
| Any | Payment card number | Checked | 13–19 digits, spaces or dashes allowed, a known issuer prefix (ISO/IEC 7812), Luhn |
|
||||
| Any | IBAN | Checked | country code, length for that country, ISO 13616 mod 97 |
|
||||
| Any | SWIFT/BIC | Needs a word | 8 or 11 characters, a valid country code in positions 5–6 |
|
||||
| Any | Email addresses, in bulk | Checked | a count of distinct addresses (a customer list leaving), not one address |
|
||||
| Any | Phone numbers, in bulk | Needs a word | a count of distinct numbers in international or national form |
|
||||
| Any | Date of birth | Needs a word | a date beside "born", "DOB", "date of birth" and their translations |
|
||||
| Any | Passport number | Needs a word | the formats of the issuing countries in this table |
|
||||
| Any | Private key | Checked | a PEM or OpenSSH private-key block |
|
||||
| Any | Cloud and service credentials | Checked | the published prefixes and lengths: AWS access key IDs, GitHub tokens, Slack tokens, Stripe live secret keys, Google API keys |
|
||||
| US | Social Security number | Checked | `AAA-GG-SSSS`, or nine digits with a word; never area 000, 666 or 9xx, group 00, serial 0000 |
|
||||
| US | ITIN | Checked | 9XX-GG-SSSS with the IRS's group ranges |
|
||||
| US | EIN | Needs a word | a valid IRS prefix and seven digits |
|
||||
| US | Bank routing number (ABA) | Needs a word | nine digits, a valid Federal Reserve prefix, the 3-7-1 checksum |
|
||||
| US | Driver's license | Needs a word | each state's published format |
|
||||
| US | Medicare Beneficiary Identifier | Checked | CMS's 11-character pattern and excluded letters |
|
||||
| US | National Provider Identifier | Needs a word | ten digits, Luhn over the `80840` prefix |
|
||||
| US | DEA registration number | Checked | two letters, seven digits, DEA's check digit |
|
||||
| UK | National Insurance number | Checked | two letters (HMRC's excluded prefixes), six digits, A–D |
|
||||
| UK | NHS number | Checked | ten digits, mod 11 |
|
||||
| UK | Unique Taxpayer Reference | Needs a word | ten digits |
|
||||
| Canada | Social Insurance Number | Checked | nine digits, Luhn |
|
||||
| Australia | Tax File Number | Checked | weighted mod 11 |
|
||||
| Australia | Medicare number | Checked | ten digits, weighted check digit |
|
||||
| EU | Germany: tax ID (Steuer-ID) | Checked | eleven digits, ISO 7064 MOD 11,10 |
|
||||
| EU | Germany: ID card number | Checked | nine characters, the 7-3-1 check digit |
|
||||
| EU | France: social security number (NIR) | Checked | fifteen characters, mod 97 key |
|
||||
| EU | Spain: DNI and NIE | Checked | eight digits and the mod 23 letter |
|
||||
| EU | Italy: codice fiscale | Checked | sixteen characters, the check letter |
|
||||
| EU | Netherlands: BSN | Checked | nine digits, the eleven test |
|
||||
| EU | Belgium: national number | Checked | eleven digits, mod 97 |
|
||||
| EU | Poland: PESEL | Checked | eleven digits, weighted check digit |
|
||||
| EU | Sweden: personnummer | Checked | a date, three digits and a Luhn check digit |
|
||||
| EU | Denmark: CPR number | Needs a word | a valid date and four digits |
|
||||
| EU | Finland: personal identity code | Checked | a date, a century sign, three digits, the mod 31 character |
|
||||
| EU | Ireland: PPS number | Checked | seven digits, one or two letters, mod 23 |
|
||||
| EU | Portugal: NIF | Checked | nine digits, mod 11 |
|
||||
| EU | Austria: social insurance number | Checked | ten digits, weighted check digit |
|
||||
| Europe | Norway: national identity number | Checked | eleven digits, two mod 11 check digits |
|
||||
| Europe | Switzerland: AHV number | Checked | `756`, then ten digits, EAN-13 check |
|
||||
| Asia | India: Aadhaar | Checked | twelve digits, Verhoeff |
|
||||
| Asia | India: PAN | Needs a word | five letters, four digits, a letter |
|
||||
| Asia | China: resident ID | Checked | eighteen characters, ISO 7064 MOD 11-2 |
|
||||
| Asia | Japan: My Number | Checked | twelve digits, weighted check digit |
|
||||
| Asia | Singapore: NRIC and FIN | Checked | a letter, seven digits, the check letter |
|
||||
| Asia | South Korea: resident registration number | Needs a word | thirteen digits with a valid date |
|
||||
| Americas | Brazil: CPF and CNPJ | Checked | two mod 11 check digits |
|
||||
| Americas | Mexico: CURP | Checked | eighteen characters, the check digit |
|
||||
| Africa | South Africa: ID number | Checked | thirteen digits with a valid date, Luhn |
|
||||
| Any | Word list | — | a list the organization maintains, counted |
|
||||
| Any | Pattern | — | the organization's own regular expression, counted |
|
||||
|
||||
Some protected data has no number to find: health conditions, religion,
|
||||
union membership, sexual orientation, criminal records. No detector claims to
|
||||
recognize those; a **word list** is how an organization covers its own terms
|
||||
for them, and the console offers editable starting lists (medical terms,
|
||||
diagnosis codes as ICD-10 patterns) rather than presenting them as detection.
|
||||
|
||||
**Templates**, so a policy doesn't pick forty detectors one at a time. Each
|
||||
is a named set, editable once added, and named for what it finds, never for a
|
||||
law: *Payment cards and bank accounts*, *US personal identifiers*, *UK
|
||||
personal identifiers*, *EU national identifiers*, *Health identifiers* (the NHS number, the US Medicare Beneficiary
|
||||
Identifier, NPI and DEA numbers, the Australian Medicare number), *Credentials and keys*, *Contact lists*.
|
||||
|
||||
The catalog grows by table entry: a new identifier is one row, one check
|
||||
function and its published examples as tests.
|
||||
|
||||
What the detectors read: the subject, every text and HTML part (as text),
|
||||
and attachments whose detected type is text (`text/*`, CSV, JSON, XML).
|
||||
Office documents (DOCX, XLSX, PPTX, ODT, ODS, ODP) are read too (settled
|
||||
answer 2): they're ZIP files of XML, unpacked and read in-house with limits on
|
||||
unpacked size and entry count. PDF files count as **can't be inspected** in
|
||||
this version, so a policy can still act on them. Inspection stops at
|
||||
a limit per message (proposed 10 MB of text), and what's past it counts as
|
||||
can't be inspected too.
|
||||
|
||||
### 2.4 Actions
|
||||
|
||||
**Transport actions** (both kinds): add a disclaimer (text and HTML, top or
|
||||
bottom, once per thread), add or remove a header, prefix the subject, add a
|
||||
recipient (a copy), redirect to other recipients, refuse with a text, send
|
||||
through a chosen route (an existing `x:MtaVirtualQueue`).
|
||||
|
||||
**DLP actions**, exactly one per DLP rule:
|
||||
|
||||
| Action | Sender sees | Message |
|
||||
|---|---|---|
|
||||
| **Block** | SMTP `550 5.7.1` with the rule's notice text; in the webmail, the notice in the send dialog | Not accepted; nothing is stored |
|
||||
| **Warn** | The notice and, once they give a reason, can send anyway | Sent after an override; the reason is audited |
|
||||
| **Hold for review** | Accepted with "held for review"; a notice mail from the server if the rule asks | Waits in the queue for a reviewer |
|
||||
|
||||
When several DLP rules match, the strictest wins: block, then hold, then warn.
|
||||
|
||||
### 2.5 Warn and override
|
||||
|
||||
**Webmail (JMAP).** The first submission fails with a new error type,
|
||||
`inbuxa:dlpWarning`, carrying the matched rules' names and notice texts (never
|
||||
the matched text). The webmail shows them and asks for a reason; it
|
||||
resubmits with `inbuxa:dlpOverride: {"reason": "..."}` on the
|
||||
`EmailSubmission` create (capability `urn:inbuxa:jmap`). The server passes the
|
||||
reason into the local SMTP session as trusted session data, not as a header,
|
||||
so it can't be forged from the message. An override covers only the rules
|
||||
that warned; if a block or hold rule also matches, that still applies.
|
||||
|
||||
**Mail apps (SMTP).** They show whatever text the server returns, so the
|
||||
refusal says how to override: `550 5.7.1 <notice>. To send anyway, start the
|
||||
subject with [override: your reason]`. On the next attempt the engine strips
|
||||
the tag before DKIM signing, and records the reason (settled answer 1).
|
||||
|
||||
A block's refusal uses the same error path with `inbuxa:dlpBlocked` in the
|
||||
webmail.
|
||||
|
||||
### 2.6 Hold for review
|
||||
|
||||
A held message is queued normally but **not scheduled**: its due time is set
|
||||
to never, and a review record `inbuxa:HeldMessage` (queue id, sender,
|
||||
recipients, subject, size, matched rules and counts, held at, expires at) is
|
||||
written in inbuxa's own subspace. The queue's stored format is untouched, so a
|
||||
node still on the previous version during a rolling upgrade reads the message
|
||||
fine and simply never sends it.
|
||||
|
||||
The sender gets `250 2.0.0 Held for review`, and the rule may send them a
|
||||
notice mail. The message stays in their Sent folder as usual.
|
||||
|
||||
A reviewer, under **Management › Compliance › Held mail**:
|
||||
|
||||
- sees the list, and opens one to read it (each opening is audited, like any
|
||||
access to someone else's mail);
|
||||
- **releases** it with a reason: it's scheduled at once and delivered as
|
||||
normal;
|
||||
- **rejects** it with a reason: it's removed from the queue and the sender
|
||||
gets a notice with the reviewer's note, not their name.
|
||||
|
||||
Unreviewed mail is rejected back to the sender after **7 days**, with a
|
||||
notice (settled answer 5); the number is a setting. Emails › Queue shows held mail as held and refuses **Retry** on it, so
|
||||
nobody can deliver it around the review. The sender can't unsend it either
|
||||
once it's held (the webmail says so).
|
||||
|
||||
Held messages count against no one's quota. Each held message and each
|
||||
decision is in the audit log.
|
||||
|
||||
**As built (phase 3).** Holding uses the queue's own future-release
|
||||
mechanism: the message is queued with its release a century off, every
|
||||
recipient's retry, notice and expiry pushed with it, so the stored format
|
||||
doesn't change. Release puts each recipient due now, keeps the gap to its
|
||||
next notice, and counts its lifetime from the release. The review record
|
||||
(`inbuxa:HeldMessage`, under `R` `h` + queue id) holds the sender,
|
||||
recipients, subject, size, rules and counts. Transport rules still apply to
|
||||
held mail, so what's released is what would have gone out. The daily
|
||||
clean-up rejects what's past its 7 days (recorded as the server's doing).
|
||||
`preview` returns the text (64 KB) only when asked for, and each read is
|
||||
recorded as `blobAccess`. Emails › Queue refuses to change or delete held
|
||||
mail, and the sender can't unsend it. How many days held mail waits is
|
||||
`inbuxa:DlpSettings.keepHeldDays`, 1 to 90, 7 by default; each held message
|
||||
keeps the days it was given.
|
||||
|
||||
### 2.7 What's recorded
|
||||
|
||||
Every DLP match writes one audit record, and **never the matched text**:
|
||||
the log would otherwise become a second copy of what the policy was keeping
|
||||
in. A card number isn't written, even masked.
|
||||
|
||||
**As built (phase 2f).** The actor is the sender (they sent it; filtering by
|
||||
sender is what a reviewer wants), the action `create`, the target kind
|
||||
`message`. The details say what happened, where to, and each rule with its
|
||||
detectors' counts: `DLP warned, to elsewhere.org: "Cards leaving"
|
||||
(payment-card 1)`. A block or an unanswered warning is recorded as refused
|
||||
(`inbuxa:dlpBlocked`, `inbuxa:dlpWarning`); an override as a success, with
|
||||
the sender's reason. No new audit action was added: an older node reading a
|
||||
record with an action it doesn't know fails its daily clean-up, so a new
|
||||
action would make rolling back unsafe.
|
||||
|
||||
Transport rules that refuse a message or change where it goes (redirect, add
|
||||
a recipient, route) record the rule and what it did the same way, the actor
|
||||
being the sender, or `system:mail-flow` for incoming mail. **As built
|
||||
(phase 2g)**, rules that only change wording or headers (a disclaimer, a
|
||||
header, a subject prefix) write nothing: a banner rule would otherwise write
|
||||
a record for every message, kept for the audit log's two years. Unmatched
|
||||
mail writes nothing.
|
||||
|
||||
### 2.8 Permissions and who does what
|
||||
|
||||
New permissions (ids from 674):
|
||||
|
||||
| Permission | Gives |
|
||||
|---|---|
|
||||
| `sysMailRuleGet` / `Update` | See / change transport rules |
|
||||
| `sysDlpPolicyGet` / `Update` | See / change DLP rules |
|
||||
| `sysDlpReviewGet` | See held mail and open it |
|
||||
| `sysDlpReviewUpdate` | Release or reject held mail |
|
||||
|
||||
**Administrator** has all. **Compliance Officer** (server-level) has
|
||||
`sysDlpPolicyGet`, `sysDlpReviewGet` and `sysDlpReviewUpdate`: officers see
|
||||
the rules and review held mail, administrators edit (settled answer 4), so
|
||||
"officers change no setting" stays true. Tenant roles get none: rules and the
|
||||
review queue are server-level (settled answer 3).
|
||||
|
||||
### 2.9 Privacy catalog
|
||||
|
||||
New entries, so the catalog check passes: `inbuxa:MailRule` (administrator
|
||||
identities), `inbuxa:HeldMessage` (sender, recipients, subject: held until
|
||||
reviewed or expired, then removed), and the held message's content in the
|
||||
queue (content, the sender's and correspondents'). The DLP audit records are
|
||||
covered by the audit log's entry.
|
||||
|
||||
### 2.10 Mixed versions and clusters
|
||||
|
||||
Rules and review records live in the shared data store, so every node sees the
|
||||
same ones. During a rolling upgrade a node still on the old version doesn't
|
||||
check mail against rules; the Overview can't tell. The console says so when
|
||||
nodes report different versions, and the release notes say to enable DLP
|
||||
rules after every node is upgraded.
|
||||
|
||||
### 2.11 Cost
|
||||
|
||||
Rules are compiled once when they change (regexes, word lists as an
|
||||
Aho-Corasick automaton) and shared by every session. Detectors only run on
|
||||
mail that some enabled rule could match (direction, sender, recipient checks
|
||||
first). The inspection limit caps the worst case.
|
||||
|
||||
## 3. Console
|
||||
|
||||
- **Management › Compliance › Data loss prevention**: DLP rules, a form with
|
||||
conditions, exceptions, detectors and the action; the notice text; what
|
||||
matched in the last 30 days (from the audit log).
|
||||
- **Management › Compliance › Held mail**: the review queue.
|
||||
- **Settings › Mail flow › Rules** (with the settings reorganization's
|
||||
approved order): transport rules, same form, ordered, with **Stop
|
||||
processing**.
|
||||
|
||||
Every form previews the rule in words ("If a recipient is outside and the
|
||||
message contains 5 or more card numbers, hold it for review").
|
||||
|
||||
## 4. Webmail (ihasmail-inbuxa)
|
||||
|
||||
- A warning dialog: the notice, a reason field, **Send anyway** and **Edit
|
||||
message**.
|
||||
- A block dialog with the notice.
|
||||
- A held message shows as **Held for review** in Sent, and its undo is gone.
|
||||
|
||||
## 5. Tests
|
||||
|
||||
Unit: each detector against valid and near-miss numbers (Luhn-failing cards,
|
||||
IBANs with a wrong check, SSN areas 000/666/9xx), word lists, regexes, the
|
||||
inspection limit, the can't-be-inspected cases, rule order and stop
|
||||
processing. Integration (`tests/src/smtp/`, `tests/src/jmap/`): block, warn
|
||||
and override over SMTP and JMAP, hold then release and reject, expiry,
|
||||
Retry refused on held mail, audit records carrying no matched text, a
|
||||
message queued by a node without the engine (held message format unchanged).
|
||||
|
||||
## 6. Phases
|
||||
|
||||
1. This spec, approved.
|
||||
2. Engine, conditions, the detector framework and the catalog in §2.3,
|
||||
Office text extraction, transport actions; DLP block and warn over SMTP and
|
||||
JMAP; audit records; catalog entries. The detector catalog may land in
|
||||
more than one PR (by region), each with its published test vectors.
|
||||
3. Hold for review: review records, release, reject, expiry, queue guard.
|
||||
4. Console: DLP rules, held mail, mail flow rules.
|
||||
5. Webmail dialogs; docs; a row in `inbuxa-drafts/divergence-log.md`.
|
||||
|
||||
Each phase is its own PR with tests; releases as John decides.
|
||||
|
||||
## Known gaps
|
||||
|
||||
- Mail a user's own filter forwards automatically to an outside address isn't
|
||||
checked in this version (it leaves as generated mail, not a submission).
|
||||
- What a mail app keeps in its own Sent folder, or sends through another
|
||||
server, is outside what this server sees.
|
||||
- Detectors find formats, not meaning: a card number in a harmless test
|
||||
message matches; a number written in words doesn't.
|
||||
|
||||
## Settled
|
||||
|
||||
John, 2026-09-28, all six as recommended, with 6 widened:
|
||||
|
||||
1. **Override from mail apps**: the `[override: reason]` subject tag, stripped
|
||||
before sending (§2.5).
|
||||
2. **Office and PDF**: Office documents are read in this version; PDF counts
|
||||
as can't be inspected (§2.3).
|
||||
3. **Tenants**: server-level rules only, with a Tenant condition (§2.2, §2.8).
|
||||
4. **Who edits DLP rules**: administrators; compliance officers see the rules
|
||||
and review held mail (§2.8).
|
||||
5. **Unreviewed held mail**: rejected back to the sender after 7 days, with a
|
||||
notice (§2.6).
|
||||
6. **Detectors**: the five proposed "and any other recognized and protected
|
||||
PII", which §2.3 turns into a catalog of identifiers with published formats
|
||||
and checks, plus templates. Data with no number to find (health,
|
||||
religion...) is covered by word lists, not claimed as detection.
|
||||
@@ -0,0 +1,324 @@
|
||||
# Feature spec: journaling
|
||||
|
||||
Status: **approved 2026-09-28**, with the answers under [Settled](#settled).
|
||||
Not a rebuild of an upstream feature, so it has no line in SPEC.md §4's table.
|
||||
Rule IDs: **JR-**.
|
||||
|
||||
## Provenance
|
||||
|
||||
Written for the record SPEC.md §3 rule 3 asks for. Sources, and nothing else:
|
||||
|
||||
| Source | License | Used for |
|
||||
|---|---|---|
|
||||
| This repository at `94a3a76` (2026-09-28): `crates/smtp/src/inbound/data.rs`, `inbound/rcpt.rs`, `queue/spool.rs`, `outbound/delivery.rs`, `crates/common/src/network/mta.rs`, `crates/features/src/{hold,audit,mailflow,undelete}`, `crates/store/src/write/{mod,blob}.rs`, `crates/jmap/src/inbuxa/hold_export.rs` | AGPL-3.0-only | Where every message passes, what the envelope holds, how holds keep blobs, how the audit chain and hold export work |
|
||||
| `inbuxa-drafts/queue/journaling.md` | Own | What John asked for, and the gaps to settle |
|
||||
| The DLP and mail flow rules spec, the audit-hold-lock spec, the personal-data catalog spec | Own | Conditions, the audit log, legal holds, roles, the catalog check |
|
||||
| RFC 5321, RFC 3461 (DSN, ORCPT), RFC 2046 (`message/rfc822`), RFC 5322 | Public | The envelope, the original recipient of an expanded list, the report's shape |
|
||||
|
||||
No Enterprise-only file or snippet was used, and no third-party journaling
|
||||
product or report format was consulted: the journal report below is our own
|
||||
layout of the SMTP envelope around the untouched message.
|
||||
|
||||
## What it is
|
||||
|
||||
A **journal** is a copy of each message the server handles, captured in
|
||||
transit with its **envelope** (the real sender and every recipient, including
|
||||
Bcc and the members of lists), kept where nobody can change or remove it
|
||||
until its retention ends, or sent to an outside archive. It sits beside two
|
||||
things that exist:
|
||||
|
||||
- **Legal hold** keeps what's in chosen mailboxes, including what their owners
|
||||
delete. It starts when a hold is placed and can't see Bcc or what was sent
|
||||
from a mailbox that no longer exists.
|
||||
- **The audit log** records what people and the server did, never the mail.
|
||||
|
||||
A journal answers the question neither can: *what went through, to whom,
|
||||
from the day it was turned on*.
|
||||
|
||||
**Out of scope**: journaling mail stored before it's turned on, files,
|
||||
calendar and contacts, IMAP APPEND (a mail app saving to its own Sent folder
|
||||
sends nothing), and mail a mail app sends through another server.
|
||||
|
||||
Nothing in code, docs, UI text or output claims the product meets a legal or
|
||||
regulatory standard. The pages say what's captured, where it's kept and for
|
||||
how long.
|
||||
|
||||
## 1. What exists today
|
||||
|
||||
Checked by reading the code at `94a3a76`:
|
||||
|
||||
| Need | Today |
|
||||
|---|---|
|
||||
| One place all mail passes | `MessageWrapper::queue` (`queue/spool.rs` ~L375). SMTP, JMAP submission (`jmap/src/submission/set.rs` builds a local session and runs `queue_message`), inbound mail, Sieve redirects and vacation replies, and DSNs all queue through it. Local and remote delivery both start from the queue. |
|
||||
| The envelope | At queue time: `mail_from`, every `rcpt_to` (Bcc included), the authenticated account with its groups and tenant, the queue id. Lists are **already expanded** at RCPT (`rcpt_resolve` → `RcptResolution::Expand`, `inbound/rcpt.rs`); the list address survives as each member's ORCPT (`dsn_info`). |
|
||||
| A copy out | Sieve at DATA, milters and MTA hooks can send one, but all run **before** DLP and transport rules, so they miss recipients the rules add, and a Sieve copy carries no envelope. |
|
||||
| Keeping a blob nobody can delete | No "undeletable" flag. Blobs are content-addressed (can't be edited); a `BlobLink::Temporary { until }` keeps one until `until`. Legal hold uses `until` = year 9999. |
|
||||
| A record nobody can quietly change | The audit log's per-node SHA-256 chain (`features/src/audit/log.rs`): each entry carries `prev`, the head is asserted on append, purge leaves a floor hash, `verify` walks it. |
|
||||
| Export | Hold export (LH-12): a ZIP of `.eml` files, `manifest.csv` with a SHA-256 per file, `manifest.sha256`, capped at 2 GiB. |
|
||||
| Conditions by sender, recipient, group, tenant | The mail flow engine (`features/src/mailflow/engine.rs`), at DATA. |
|
||||
|
||||
## 2. Design
|
||||
|
||||
### 2.1 Where the copy is taken (JR-1, JR-2)
|
||||
|
||||
**JR-1.** The journal is taken in `MessageWrapper::queue`, after the message
|
||||
is spooled, behind one `// inbuxa:` marked block. That's after DLP and
|
||||
transport rules, so the envelope is the one the message actually leaves or
|
||||
arrives with, and it covers every path that queues mail.
|
||||
|
||||
**JR-2.** What isn't journaled: journal reports themselves (they carry a
|
||||
queue flag, so a report to an outside archive can't journal itself), and the
|
||||
server's own DMARC and TLS reports. DSNs and Sieve redirects and vacation
|
||||
replies are journaled (question 4). A message **refused** at DATA (DLP block,
|
||||
a transport rule's refusal) was never accepted and isn't journaled; the
|
||||
audit log already records it. A message **held** for DLP review is journaled
|
||||
when it's queued, which is when it's held, with the hold noted in the entry.
|
||||
|
||||
### 2.2 The journal report (JR-3, JR-4)
|
||||
|
||||
**JR-3.** Each copy is a **journal report**: a new message whose first part
|
||||
is `text/plain`, one field a line:
|
||||
|
||||
```
|
||||
Sender: alice@example.com
|
||||
Signed in as: alice@example.com
|
||||
Subject: Q3 figures
|
||||
Message-ID: <…>
|
||||
Queue ID: 1a2b3c…
|
||||
Received: 2026-09-28T14:03:11Z
|
||||
Direction: outgoing
|
||||
To: bank@elsewhere.example
|
||||
Cc: bob@example.com
|
||||
Bcc: carol@example.com
|
||||
Expanded: finance@example.com -> dan@example.com, erin@example.com
|
||||
Held for review: yes
|
||||
```
|
||||
|
||||
and whose second part is the message as queued, **byte for byte**, as
|
||||
`message/rfc822`. `Bcc:` lists envelope recipients that aren't in the
|
||||
message's To or Cc headers. `Expanded:` groups the members of a list under
|
||||
the list address, from their ORCPT. Recipients a transport rule added say so
|
||||
(`Added by rule: <name>`). The field names are fixed English (they're a
|
||||
record, not interface text), so a script can read them.
|
||||
|
||||
**JR-4.** One report per queued message, with the whole envelope, whatever
|
||||
the scope matched on (§2.4). A message to 40 recipients is one report, not
|
||||
40.
|
||||
|
||||
### 2.3 Where reports go (JR-5 to JR-8)
|
||||
|
||||
Each journal has a **destination** (question 1):
|
||||
|
||||
**JR-5. The built-in journal.** Records under a new prefix `J` in
|
||||
`SUBSPACE_INBUXA`: queue id, received time, direction, sender, recipients,
|
||||
the tenant(s), which journal matched, the report's blob hash and size, its
|
||||
SHA-256, and the time it may be purged. The report blob is kept by a
|
||||
`BlobLink::Temporary { until }` set to the end of its retention. There is no
|
||||
JMAP `set` or `destroy` for entries: nothing in the product changes or
|
||||
removes one before its time.
|
||||
|
||||
**JR-6. The chain.** Each entry carries the SHA-256 of the entry before it,
|
||||
one chain per node, the same construction as the audit log (and its code,
|
||||
generalized rather than copied). The console's **Check the journal** walks
|
||||
it, and every blob's hash against its entry, and says what it found. Someone
|
||||
with the server's disks can still remove data, and the chain is how that
|
||||
shows; the docs say exactly that, and don't say it can't happen.
|
||||
|
||||
**JR-7. An outside archive.** The report is queued to an address (the
|
||||
archive's journal mailbox) like any mail, with the queue's retries. A report
|
||||
the archive refuses permanently, or can't take within the queue's limit, goes
|
||||
into the built-in journal instead and raises a warning on the Overview
|
||||
(question 6). Delivery is by the ordinary queue, so TLS and routing settings
|
||||
apply; a queue route can be chosen for it.
|
||||
|
||||
**JR-8. Both**: the built-in journal and an outside archive.
|
||||
|
||||
### 2.4 Which mail: journals and their scope (JR-9 to JR-11)
|
||||
|
||||
**JR-9.** A **journal** is a named object (`inbuxa:Journal`): on or off, a
|
||||
destination, a retention, and a scope. The scope is who: **everyone**, or
|
||||
senders and recipients in chosen **accounts, groups, domains or tenants**,
|
||||
and which **direction**: outgoing, incoming, internal, any. A message is
|
||||
journaled once per journal whose scope any sender or recipient is in; two
|
||||
journals with the same destination never write the same message twice.
|
||||
|
||||
**JR-10.** **By what's in it**: a new mail flow rule action, **Journal it**,
|
||||
names a journal. The rule's conditions (detectors, words, attachments,
|
||||
headers) decide; the copy is still taken at queue time (the rule only marks
|
||||
the message). This is the rule-based journaling the queue note called
|
||||
premium; here it's one more action, not a separate tier (question 2).
|
||||
|
||||
**JR-11.** Scope is evaluated from the envelope and directory membership at
|
||||
queue time (`Server::member_of`), no message parsing, so journaling
|
||||
everything costs a lookup per recipient and one blob write per message.
|
||||
|
||||
### 2.5 Retention and legal hold (JR-12 to JR-14)
|
||||
|
||||
**JR-12.** Each journal has a retention in days (question 3). An entry keeps
|
||||
the retention it was written with: shortening a journal's retention applies
|
||||
to new entries only, so nobody can empty the journal by editing a number.
|
||||
Lengthening it applies to new entries too, and the console says so.
|
||||
|
||||
**JR-13.** Purge runs in the daily maintenance, removes entries past their
|
||||
time and drops their blob link, and leaves a floor hash so the chain still
|
||||
verifies, as the audit log does. An entry whose sender or any recipient is
|
||||
under a **legal hold** isn't purged while the hold lasts (`holds_on`, read
|
||||
uncached, as holds are everywhere).
|
||||
|
||||
**JR-14.** Deleting an account doesn't remove its journal entries; they end
|
||||
with their retention (question 7). The privacy catalog says so.
|
||||
|
||||
### 2.6 Search, reading, export (JR-15 to JR-17)
|
||||
|
||||
**JR-15.** **Management › Compliance › Journal**: search by sender,
|
||||
recipient, date range, direction, subject words (from the report's header
|
||||
fields, not the body: no full-text index of the journal in this version).
|
||||
Results list the envelope; **Read…** opens the report.
|
||||
|
||||
**JR-16.** **Export** a search as a ZIP in the hold export's shape: the
|
||||
reports as `.eml`, `manifest.csv` with the envelope columns and a SHA-256
|
||||
per file, `manifest.sha256`, the same 2 GiB cap. Export runs as a task and
|
||||
the result is a blob owned by the person who asked for it.
|
||||
|
||||
**JR-17.** Every search, read and export is in the audit log, with who and
|
||||
the search terms; so is every change to a journal.
|
||||
|
||||
### 2.7 Permissions (JR-18)
|
||||
|
||||
**JR-18.** New permissions after the DLP set (680 onward):
|
||||
`sysJournalGet` / `sysJournalUpdate` (see and change journals),
|
||||
`sysJournalSearch` (search and read entries), `sysJournalExport`. Superuser
|
||||
only, by default. The officer grant audience adds Get, Search and Export to
|
||||
the Compliance Officer; administrators configure journals but don't read
|
||||
them unless granted Search (question 5). Journals are server-level, with a
|
||||
tenant scope, as DLP rules are; nobody in a tenant reaches them.
|
||||
|
||||
### 2.8 Privacy catalog
|
||||
|
||||
New objects get catalog entries (`resources/privacy/catalog.toml`):
|
||||
`inbuxa:Journal` (none), `inbuxa:JournalEntry` (mail content and envelope,
|
||||
kept for the journal's retention, access audited, not erased with the
|
||||
account). `privacy-check.py` enforces it.
|
||||
|
||||
### 2.9 Mixed versions, clusters, rollback
|
||||
|
||||
- Entries and journals live in the shared data store; report blobs in the
|
||||
blob store. A **node-local** blob store (FileSystem, or RocksDB/SQLite as
|
||||
the blob store) on a cluster means a node's journal lives on that node;
|
||||
the console warns when journaling is on and the blob store isn't shared.
|
||||
- During a rolling upgrade a node on the old version doesn't journal. The
|
||||
console says so when nodes report different versions; the release notes
|
||||
say to turn journals on after every node is upgraded.
|
||||
- Rollback: the new prefix and the queue flag are ignored by an older
|
||||
version; nothing in the queue's archived format changes (the "journal
|
||||
report" flag rides in the existing message flags if one is free, else in
|
||||
a side key by queue id; checked in phase 2 before writing code).
|
||||
|
||||
### 2.10 Cost
|
||||
|
||||
One extra blob per journaled message (the report wraps the original, so it
|
||||
doesn't share its hash), plus one small record. The console shows the
|
||||
journal's size and growth per day on the journal page, from the entries.
|
||||
|
||||
## 3. Console
|
||||
|
||||
- **Management › Compliance › Journaling**: journals (name, scope,
|
||||
destination, retention, on/off), described in words like DLP rules
|
||||
("Journal all mail to and from Finance into the built-in journal, kept
|
||||
7 years"); **Check the journal**.
|
||||
- **Management › Compliance › Journal**: search, read, export.
|
||||
- The mail flow rule editor gains **Journal it**.
|
||||
- The Overview warns about undelivered outside reports (JR-7) and a
|
||||
node-local blob store (§2.9).
|
||||
|
||||
## 4. Webmail
|
||||
|
||||
Nothing. People aren't told a message was journaled, as they aren't told
|
||||
about legal hold; the docs say journaling exists and what it captures.
|
||||
|
||||
## 5. Tests
|
||||
|
||||
Unit: the report's fields (Bcc computed from headers, list expansion from
|
||||
ORCPT, rule-added recipients), scope matching, retention arithmetic, the
|
||||
chain. Integration (`tests/src/system/`): SMTP and JMAP sends, inbound
|
||||
mail, internal mail, a list and a Bcc recipient, a DLP-held message, a
|
||||
Sieve redirect; the report equals the queued bytes; no `set`/`destroy`;
|
||||
shortening retention doesn't touch existing entries; a hold stops purge;
|
||||
account deletion leaves entries; an outside archive that refuses falls back
|
||||
to the built-in journal; export manifest hashes; audit records for search,
|
||||
read, export.
|
||||
|
||||
## 6. Phases
|
||||
|
||||
1. This spec, approved.
|
||||
2. Capture at the queue, the report, the built-in journal with its chain,
|
||||
retention and purge, holds; `inbuxa:Journal` and `inbuxa:JournalEntry`;
|
||||
catalog entries; tests.
|
||||
3. Outside archive and the fallback; **Journal it** in mail flow rules.
|
||||
4. Search, read and export (a task), audit records.
|
||||
5. Console pages; docs; a row in `inbuxa-drafts/divergence-log.md`.
|
||||
|
||||
Each phase is its own PR with tests; releases as John decides. Like DLP, it
|
||||
stays out of production until John says.
|
||||
|
||||
## As built
|
||||
|
||||
Phase 2 (`feature/journal-capture`), where it differs from the design or
|
||||
fills in what it left open:
|
||||
|
||||
- **The chain** is the journal's own (`crates/features/src/journal/
|
||||
entries.rs`), not the audit log's code shared. Entries expire out of chain
|
||||
order (each keeps its journal's retention, and holds keep some longer), so
|
||||
a link names its entry by SHA-256 instead of holding it: purging removes
|
||||
the entry, its indexes and its report's blob link, and writes a purge
|
||||
marker; the link stays. An entry missing without a marker is a broken
|
||||
chain. Purged links at a chain's start are cleared and a floor recorded,
|
||||
as the audit log does.
|
||||
- **If the copy can't be taken**, the message isn't queued: the sender gets
|
||||
a temporary failure and tries again. Nothing leaves unjournaled.
|
||||
- **The report** says `Authenticated: yes|no` instead of the signed-in
|
||||
account (the queue doesn't keep which account it was). `Added by rule`
|
||||
comes with **Journal it** in phase 3. A recipient given with an ORCPT
|
||||
that names another address counts as expanded from that address.
|
||||
- **Journal reports** the server queues carry message flag bit 48
|
||||
(`FROM_JOURNAL`); an older version ignores the bit.
|
||||
- **Permissions 680–683**: administrators get `sysJournalGet`/`Update`; the
|
||||
Compliance Officer gets `Get`, `Search` and `Export`. So that an
|
||||
administrator can still appoint an officer (and grant reading as settled
|
||||
answer 5 describes), whoever holds `sysJournalUpdate` may grant `Search`
|
||||
and `Export` without holding them; the role change is in the audit log.
|
||||
- **`inbuxa:JournalEntry`** (get, query) and **Check the journal** over
|
||||
JMAP come in phase 4 with search, so every read is audited from the first
|
||||
version that allows one. Phase 2 has `inbuxa:Journal` only.
|
||||
- **Outside archives** (a journal's destination) come in phase 3; every
|
||||
journal writes to the built-in journal until then.
|
||||
|
||||
## Known gaps
|
||||
|
||||
- A message a person saves to Sent over IMAP, or sends through another
|
||||
server, never reaches the queue.
|
||||
- Mail stored before journaling is on isn't journaled (legal hold covers
|
||||
mailboxes).
|
||||
- Search reads envelope and header fields, not bodies.
|
||||
- Group accounts (`GroupAccount`) resolve as one account, not members; their
|
||||
mail is journaled under the group's address.
|
||||
|
||||
## Settled
|
||||
|
||||
John, 2026-09-28, all seven as recommended:
|
||||
|
||||
1. **Destinations**: the built-in journal, an outside archive by address, or
|
||||
both, per journal (JR-5, JR-7, JR-8).
|
||||
2. **Scope**: everyone, or chosen accounts, groups, domains and tenants by
|
||||
direction, plus a **Journal it** rule action; no standard/premium split
|
||||
(JR-9, JR-10).
|
||||
3. **Retention**: no default; 30 days to 10 years, picked when a journal is
|
||||
turned on; existing entries keep theirs (JR-12).
|
||||
4. **Which mail**: everything queued, including DSNs, Sieve redirects and
|
||||
vacation replies, except DMARC/TLS reports and journal reports (JR-2).
|
||||
5. **Who reads it**: administrators configure; Compliance Officers search,
|
||||
read and export; administrators read only if granted Search (JR-18).
|
||||
6. **An outside archive that won't take a report**: kept in the built-in
|
||||
journal, with a warning (JR-7).
|
||||
7. **Deleted accounts**: journal entries stay until their retention ends,
|
||||
and the catalog says so (JR-14).
|
||||
Binary file not shown.
@@ -79,6 +79,63 @@ lockedAt = ["metadata"]
|
||||
lockedBy = ["identifier"]
|
||||
delegates = ["identifier"]
|
||||
|
||||
[object."inbuxa:DlpSettings"]
|
||||
file = "inbuxa_dlp_settings.rs"
|
||||
default = "none"
|
||||
|
||||
[object."inbuxa:HeldMessage"]
|
||||
file = "inbuxa_held_message.rs"
|
||||
default = "none"
|
||||
whose = ["holder", "correspondent"]
|
||||
where = ["data-store", "blob-store"]
|
||||
scope = "server"
|
||||
retention = "object-life"
|
||||
[object."inbuxa:HeldMessage".properties]
|
||||
sender = ["identifier", "contact"]
|
||||
recipients = ["identifier", "contact"]
|
||||
subject = ["content"]
|
||||
preview = ["content"]
|
||||
note = ["content"]
|
||||
counts = ["metadata"]
|
||||
|
||||
[object."inbuxa:MailRule"]
|
||||
file = "inbuxa_mail_rule.rs"
|
||||
default = "none"
|
||||
whose = ["administrator", "holder", "correspondent"]
|
||||
where = ["data-store"]
|
||||
scope = "server"
|
||||
retention = "unbounded"
|
||||
[object."inbuxa:MailRule".properties]
|
||||
name = ["content"]
|
||||
description = ["content"]
|
||||
conditions = ["contact", "content"]
|
||||
exceptions = ["contact", "content"]
|
||||
actions = ["contact", "content"]
|
||||
createdBy = ["identifier"]
|
||||
|
||||
[object."inbuxa:Journal"]
|
||||
file = "inbuxa_journal.rs"
|
||||
default = "none"
|
||||
whose = ["administrator"]
|
||||
where = ["data-store"]
|
||||
scope = "server"
|
||||
retention = "unbounded"
|
||||
[object."inbuxa:Journal".properties]
|
||||
name = ["content"]
|
||||
description = ["content"]
|
||||
createdBy = ["identifier"]
|
||||
|
||||
[object."inbuxa:SecurityAcceptance"]
|
||||
file = "inbuxa_security_acceptance.rs"
|
||||
default = "none"
|
||||
whose = ["administrator"]
|
||||
where = ["data-store"]
|
||||
scope = "server"
|
||||
retention = "object-life"
|
||||
[object."inbuxa:SecurityAcceptance".properties]
|
||||
note = ["content"]
|
||||
acceptedBy = ["identifier"]
|
||||
|
||||
[object."inbuxa:LegalHold"]
|
||||
file = "inbuxa_legal_hold.rs"
|
||||
default = "none"
|
||||
@@ -374,6 +431,19 @@ captures = ["x:Email.maxMaskedAddresses"]
|
||||
leaves_host = false
|
||||
written_by = ["crates/features/src/masked_email/data.rs"]
|
||||
|
||||
# Journaling (journaling spec, JR-5, JR-14): a copy of each message a
|
||||
# journal takes, with its envelope, kept for the journal's retention even
|
||||
# after the account is deleted, and longer while a legal hold covers
|
||||
# someone on it.
|
||||
[source."journal"]
|
||||
categories = ["content", "identifier", "contact", "metadata"]
|
||||
whose = ["holder", "correspondent"]
|
||||
where = ["data-store", "blob-store"]
|
||||
scope = "server"
|
||||
retention = { setting = "inbuxa:Journal.retentionDays" }
|
||||
leaves_host = false
|
||||
written_by = ["crates/features/src/journal/entries.rs", "crates/smtp/src/queue/journal.rs"]
|
||||
|
||||
[source."outbound-reports"]
|
||||
categories = ["network", "identifier", "content"]
|
||||
whose = ["correspondent"]
|
||||
|
||||
Binary file not shown.
@@ -1 +1 @@
|
||||
wDJZ1KdKs21tjHD-UBI9R_XwPEET7Iul8XEXbPlgBPE
|
||||
0Ay1tm9k_D94V7sLFfK7pIxwt3QdfYK_VBNs-rLGjhs
|
||||
@@ -1,7 +1,10 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use crate::utils::server::TestServer;
|
||||
@@ -286,6 +289,30 @@ pub async fn test(test: &TestServer) {
|
||||
not_valid_before + length / 2,
|
||||
task.due_timestamp() as i64
|
||||
);
|
||||
|
||||
// inbuxa: renewing while a valid certificate already covers the names
|
||||
// (say, one stored by hand before the domain went automatic) schedules
|
||||
// the renewal for when it falls due. It used to end the task for good.
|
||||
let rescheduled = test
|
||||
.server
|
||||
.acme_renew(tls_domain_id)
|
||||
.await
|
||||
.ok()
|
||||
.expect("a renewal that isn't due yet to be rescheduled, not to fail");
|
||||
assert!(
|
||||
matches!(
|
||||
rescheduled.as_slice(),
|
||||
[Task::AcmeRenewal(TaskDomainManagement { domain_id, .. })] if *domain_id == tls_domain_id
|
||||
),
|
||||
"Expected one rescheduled ACME renewal, found: {:?}",
|
||||
rescheduled
|
||||
);
|
||||
assert_eq!(
|
||||
rescheduled[0].due_timestamp() as i64,
|
||||
not_valid_before + length / 2,
|
||||
"The rescheduled renewal should fall due when the certificate does"
|
||||
);
|
||||
|
||||
account.registry_destroy_all(ObjectType::Certificate).await;
|
||||
account.registry_destroy_all(ObjectType::Task).await;
|
||||
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use crate::utils::server::TestServer;
|
||||
@@ -58,7 +61,7 @@ _995._tcp.pop3.example.org. IN TLSA 2 1 1
|
||||
_dmarc.example.org. IN TXT "v=DMARC1; p=reject; rua=mailto:[email protected]"
|
||||
_mta-sts.example.org. IN TXT "v=STSv1; id=12942536112359691423"
|
||||
_smtp._tls.example.org. IN TXT "v=TLSRPTv1; rua=mailto:[email protected]"
|
||||
_ua-auto-config.example.org. IN TXT "v=UAAC1; a=sha256; d=9X2mMgWAc10oSPuRKZSFBwPXEQpnxkS7SXPO8PC7euM="
|
||||
_ua-auto-config.example.org. IN TXT "v=UAAC1; a=sha256; d=ZZ35kyyCO86LM5UUTecwutQ8B+0XdZ3wJnjoYXnH0Wk="
|
||||
_validation-persist.example.org. IN TXT "pebble.letsencrypt.org; accounturi=REDACTED"
|
||||
dummy-v1-ed25519._domainkey.example.org. IN TXT "v=DKIM1; k=ed25519; h=sha256; p=REDACTED"
|
||||
dummy-v1-rsa._domainkey.example.org. IN TXT "v=DKIM1; k=rsa; h=sha256; p=REDACTED"
|
||||
|
||||
@@ -0,0 +1,471 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! Journaling (journaling spec, phase 2): journals over JMAP, the copy
|
||||
//! taken as mail is queued with its whole envelope, the report around the
|
||||
//! untouched message, retention, purge, and a chain that shows tampering.
|
||||
|
||||
use crate::utils::{
|
||||
account::Account,
|
||||
server::{TestServer, TestServerBuilder},
|
||||
smtp::SmtpConnection,
|
||||
};
|
||||
use inbuxa_features::journal::{
|
||||
Direction,
|
||||
entries::{self, Entry, EntryId},
|
||||
report,
|
||||
};
|
||||
use registry::schema::structs::{Expression, MtaStageAuth};
|
||||
use serde_json::{Value, json};
|
||||
use store::{Deserialize, write::BatchBuilder};
|
||||
|
||||
const USING: &[&str] = &[
|
||||
"urn:ietf:params:jmap:core",
|
||||
"urn:ietf:params:jmap:mail",
|
||||
"urn:ietf:params:jmap:submission",
|
||||
"urn:inbuxa:jmap",
|
||||
];
|
||||
|
||||
async fn call(account: &Account, method: &str, mut arguments: Value) -> (String, Value) {
|
||||
if arguments.get("accountId").is_none() {
|
||||
arguments["accountId"] = account.id_string().into();
|
||||
}
|
||||
let response = account
|
||||
.jmap_request(USING, json!([[method, arguments, "0"]]))
|
||||
.await;
|
||||
let call = response
|
||||
.0
|
||||
.pointer("/methodResponses/0")
|
||||
.cloned()
|
||||
.unwrap_or_else(|| panic!("{method}: {}", response.0));
|
||||
(
|
||||
call[0].as_str().unwrap_or_default().to_string(),
|
||||
call[1].clone(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Sends a message whose headers name `to`, to the envelope `rcpt_to`.
|
||||
async fn send(
|
||||
sender: &Account,
|
||||
identity: &str,
|
||||
mailbox: &str,
|
||||
to: &[&str],
|
||||
rcpt_to: &[&str],
|
||||
subject: &str,
|
||||
) -> Value {
|
||||
let (_, response) = call(
|
||||
sender,
|
||||
"Email/set",
|
||||
json!({"create": {"e": {
|
||||
"mailboxIds": {mailbox: true},
|
||||
"from": [{"email": sender.name()}],
|
||||
"to": to.iter().map(|a| json!({"email": a})).collect::<Vec<_>>(),
|
||||
"subject": subject,
|
||||
"bodyValues": {"b": {"value": "The body."}},
|
||||
"textBody": [{"partId": "b", "type": "text/plain"}]
|
||||
}}}),
|
||||
)
|
||||
.await;
|
||||
let email = response["created"]["e"]["id"]
|
||||
.as_str()
|
||||
.unwrap_or_else(|| panic!("draft: {response}"))
|
||||
.to_string();
|
||||
call(
|
||||
sender,
|
||||
"EmailSubmission/set",
|
||||
json!({"create": {"s": {
|
||||
"emailId": email,
|
||||
"identityId": identity,
|
||||
"envelope": {
|
||||
"mailFrom": {"email": sender.name()},
|
||||
"rcptTo": rcpt_to.iter().map(|a| json!({"email": a})).collect::<Vec<_>>()
|
||||
}
|
||||
}}}),
|
||||
)
|
||||
.await
|
||||
.1
|
||||
}
|
||||
|
||||
async fn all_entries(test: &TestServer) -> Vec<(EntryId, Entry)> {
|
||||
entries::list(test.server.store(), 0, u64::MAX, 10_000)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn entry_for(test: &TestServer, subject: &str) -> Option<(EntryId, Entry)> {
|
||||
all_entries(test)
|
||||
.await
|
||||
.into_iter()
|
||||
.find(|(_, e)| e.subject == subject)
|
||||
}
|
||||
|
||||
async fn report_of(test: &TestServer, entry: &Entry) -> Vec<u8> {
|
||||
let hash = entry.blob_hash().expect("blob hash");
|
||||
test.server
|
||||
.blob_store()
|
||||
.get_blob(hash.as_slice(), 0..usize::MAX)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("report blob")
|
||||
}
|
||||
|
||||
pub async fn test(test: &mut TestServer) {
|
||||
println!("Running journaling tests...");
|
||||
let admin = test.account("[email protected]");
|
||||
let sender = admin
|
||||
.create_user_account(
|
||||
"[email protected]",
|
||||
"journal-sender-secret-7101",
|
||||
"Journal sender",
|
||||
&[],
|
||||
vec![],
|
||||
)
|
||||
.await;
|
||||
let other = admin
|
||||
.create_user_account(
|
||||
"[email protected]",
|
||||
"journal-other-secret-7102",
|
||||
"Journal other",
|
||||
&[],
|
||||
vec![],
|
||||
)
|
||||
.await;
|
||||
let (_, response) = call(
|
||||
&sender,
|
||||
"Identity/set",
|
||||
json!({"create": {"i": {"name": "Sender", "email": "[email protected]"}}}),
|
||||
)
|
||||
.await;
|
||||
let identity = response["created"]["i"]["id"].as_str().unwrap().to_string();
|
||||
let (_, response) = call(
|
||||
&sender,
|
||||
"Mailbox/set",
|
||||
json!({"create": {"m": {"name": "Journal drafts"}}}),
|
||||
)
|
||||
.await;
|
||||
let mailbox = response["created"]["m"]["id"].as_str().unwrap().to_string();
|
||||
|
||||
// Nothing is journaled while there are no journals
|
||||
let response = send(
|
||||
&sender,
|
||||
&identity,
|
||||
&mailbox,
|
||||
&["[email protected]"],
|
||||
&["[email protected]"],
|
||||
"Before any journal",
|
||||
)
|
||||
.await;
|
||||
assert!(response["created"].get("s").is_some(), "{response}");
|
||||
assert!(all_entries(test).await.is_empty());
|
||||
|
||||
// Journals: checked when written, the server's own properties refused
|
||||
let (_, response) = call(
|
||||
&admin,
|
||||
"inbuxa:Journal/set",
|
||||
json!({"create": {
|
||||
"short": {"name": "Short", "enabled": true, "direction": "any",
|
||||
"scope": {"everyone": true}, "retentionDays": 29},
|
||||
"both": {"name": "Both", "enabled": true, "direction": "any",
|
||||
"scope": {"everyone": true, "accounts": [sender.id_string()]},
|
||||
"retentionDays": 365},
|
||||
"none": {"name": "None", "enabled": true, "direction": "any",
|
||||
"scope": {}, "retentionDays": 365},
|
||||
"server": {"name": "Mine", "enabled": true, "direction": "any",
|
||||
"scope": {"everyone": true}, "retentionDays": 365,
|
||||
"createdBy": "me"},
|
||||
"all": {"name": "Everything", "enabled": true, "direction": "any",
|
||||
"scope": {"everyone": true}, "retentionDays": 365},
|
||||
"out": {"name": "Sender's outgoing", "enabled": true, "direction": "outgoing",
|
||||
"scope": {"accounts": [sender.id_string()]}, "retentionDays": 3650}
|
||||
}}),
|
||||
)
|
||||
.await;
|
||||
for refused in ["short", "both", "none", "server"] {
|
||||
assert_eq!(
|
||||
response["notCreated"][refused]["type"], "invalidProperties",
|
||||
"{refused}: {response}"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
response["notCreated"]["short"]["properties"],
|
||||
json!(["retentionDays"])
|
||||
);
|
||||
assert_eq!(
|
||||
response["notCreated"]["both"]["properties"],
|
||||
json!(["scope"])
|
||||
);
|
||||
let everything = response["created"]["all"]["id"]
|
||||
.as_str()
|
||||
.unwrap_or_else(|| panic!("{response}"))
|
||||
.to_string();
|
||||
let outgoing = response["created"]["out"]["id"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.to_string();
|
||||
let (_, response) = call(&admin, "inbuxa:Journal/get", json!({"ids": null})).await;
|
||||
let list = response["list"].as_array().unwrap();
|
||||
assert_eq!(list.len(), 2, "{response}");
|
||||
assert_eq!(list[0]["name"], "Everything");
|
||||
assert_eq!(list[0]["createdBy"], "[email protected]");
|
||||
assert_eq!(list[1]["scope"]["accounts"], json!([sender.id_string()]));
|
||||
// Each node reads journals again within 30 seconds; this one at once
|
||||
inbuxa_features::journal::invalidate();
|
||||
|
||||
// Internal mail with a Bcc recipient: one entry, the whole envelope
|
||||
let response = send(
|
||||
&sender,
|
||||
&identity,
|
||||
&mailbox,
|
||||
&["[email protected]"],
|
||||
&["[email protected]", "[email protected]"],
|
||||
"Internal with Bcc",
|
||||
)
|
||||
.await;
|
||||
assert!(response["created"].get("s").is_some(), "{response}");
|
||||
let (_, entry) = entry_for(test, "Internal with Bcc")
|
||||
.await
|
||||
.expect("journaled");
|
||||
assert_eq!(entry.direction, Direction::Internal);
|
||||
assert_eq!(entry.sender, "[email protected]");
|
||||
assert!(entry.authenticated);
|
||||
assert_eq!(entry.recipients.len(), 2, "{entry:?}");
|
||||
assert_eq!(
|
||||
entry.journals.len(),
|
||||
1,
|
||||
"internal isn't outgoing: {entry:?}"
|
||||
);
|
||||
assert!(!entry.held);
|
||||
assert_eq!(entry.expires_at, entry.at + 365 * 86_400);
|
||||
let bytes = report_of(test, &entry).await;
|
||||
assert_eq!(entries::sha256(&bytes), entry.sha256);
|
||||
let text = String::from_utf8_lossy(&bytes);
|
||||
assert!(text.contains("Direction: internal\r\n"), "{text}");
|
||||
assert!(
|
||||
text.contains("To: [email protected]\r\n"),
|
||||
"{text}"
|
||||
);
|
||||
assert!(
|
||||
text.contains("Bcc: [email protected]\r\n"),
|
||||
"{text}"
|
||||
);
|
||||
let original = report::original(&bytes).expect("original part");
|
||||
let original = String::from_utf8_lossy(original);
|
||||
assert!(
|
||||
original.contains("Subject: Internal with Bcc"),
|
||||
"{original}"
|
||||
);
|
||||
assert!(original.contains("The body."), "{original}");
|
||||
assert!(!original.contains("Bcc:"), "the original is as sent");
|
||||
|
||||
// Outgoing: both journals take it, and it's kept for the longer
|
||||
let response = send(
|
||||
&sender,
|
||||
&identity,
|
||||
&mailbox,
|
||||
&["[email protected]"],
|
||||
&["[email protected]"],
|
||||
"Leaving",
|
||||
)
|
||||
.await;
|
||||
assert!(response["created"].get("s").is_some(), "{response}");
|
||||
let (leaving_id, entry) = entry_for(test, "Leaving").await.expect("journaled");
|
||||
assert_eq!(entry.direction, Direction::Outgoing);
|
||||
assert_eq!(entry.journals.len(), 2, "{entry:?}");
|
||||
assert_eq!(entry.expires_at, entry.at + 3650 * 86_400);
|
||||
|
||||
// Incoming from outside
|
||||
admin
|
||||
.registry_create_object(MtaStageAuth {
|
||||
require: Expression {
|
||||
else_: "false".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
let mut lmtp = SmtpConnection::connect().await;
|
||||
lmtp.ingest(
|
||||
"[email protected]",
|
||||
&["[email protected]"],
|
||||
"From: [email protected]\r\nTo: [email protected]\r\nSubject: Arriving\r\n\r\nHi.\r\n",
|
||||
)
|
||||
.await;
|
||||
let (_, entry) = entry_for(test, "Arriving").await.expect("journaled");
|
||||
assert_eq!(entry.direction, Direction::Incoming);
|
||||
assert!(!entry.authenticated);
|
||||
assert_eq!(entry.accounts, vec![other.id().document_id()]);
|
||||
|
||||
// The chain checks out, reports included
|
||||
let store = test.server.store();
|
||||
let blobs = test.server.blob_store();
|
||||
let reports = entries::verify(store, Some(blobs)).await.unwrap();
|
||||
assert!(reports.iter().all(|r| r.broken_at.is_none()), "{reports:?}");
|
||||
let journaled = all_entries(test).await.len() as u64;
|
||||
assert!(reports.iter().map(|r| r.entries).sum::<u64>() >= journaled);
|
||||
|
||||
// An entry changed in the store shows; put back, it checks out again
|
||||
let key = entries::content_key(leaving_id);
|
||||
let stored = store
|
||||
.get_value::<Raw>(key.clone())
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("stored entry")
|
||||
.0;
|
||||
let mut forged: Entry = serde_json::from_slice(&stored).unwrap();
|
||||
forged.recipients = vec!["[email protected]".into()];
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.set(key.class.clone(), serde_json::to_vec(&forged).unwrap());
|
||||
store.write(batch.build_all()).await.unwrap();
|
||||
let reports = entries::verify(store, None).await.unwrap();
|
||||
let broken = reports
|
||||
.iter()
|
||||
.find(|r| r.broken_at.is_some())
|
||||
.expect("broken");
|
||||
assert_eq!(
|
||||
broken.broken_at.as_deref(),
|
||||
Some(leaving_id.to_string().as_str())
|
||||
);
|
||||
assert!(
|
||||
broken
|
||||
.reason
|
||||
.as_deref()
|
||||
.unwrap_or_default()
|
||||
.contains("changed")
|
||||
);
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.set(key.class.clone(), stored.clone());
|
||||
store.write(batch.build_all()).await.unwrap();
|
||||
assert!(
|
||||
entries::verify(store, None)
|
||||
.await
|
||||
.unwrap()
|
||||
.iter()
|
||||
.all(|r| r.broken_at.is_none())
|
||||
);
|
||||
|
||||
// An entry removed without a purge shows too
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.clear(key.class.clone());
|
||||
store.write(batch.build_all()).await.unwrap();
|
||||
let reports = entries::verify(store, None).await.unwrap();
|
||||
assert!(
|
||||
reports.iter().any(|r| r
|
||||
.reason
|
||||
.as_deref()
|
||||
.unwrap_or_default()
|
||||
.contains("before its time")),
|
||||
"{reports:?}"
|
||||
);
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.set(key.class.clone(), stored);
|
||||
store.write(batch.build_all()).await.unwrap();
|
||||
|
||||
// Retention: nothing is due yet; a year on, what's kept for a hold
|
||||
// stays, the rest goes, and the chain still checks out
|
||||
let now = store::write::now();
|
||||
let purged = entries::purge(store, now, |_| false).await.unwrap();
|
||||
assert_eq!(purged.removed, 0);
|
||||
let sender_id = sender.id().document_id();
|
||||
let later = now + 400 * 86_400;
|
||||
let purged = entries::purge(store, later, |e| e.accounts.contains(&sender_id))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(purged.removed >= 1, "{purged:?}");
|
||||
assert!(purged.kept_for_hold >= 1, "{purged:?}");
|
||||
assert!(entry_for(test, "Arriving").await.is_none(), "purged");
|
||||
assert!(entry_for(test, "Internal with Bcc").await.is_some(), "held");
|
||||
assert!(entry_for(test, "Leaving").await.is_some(), "ten years");
|
||||
let reports = entries::verify(store, Some(blobs)).await.unwrap();
|
||||
assert!(reports.iter().all(|r| r.broken_at.is_none()), "{reports:?}");
|
||||
assert!(reports.iter().map(|r| r.purged).sum::<u64>() >= 1);
|
||||
|
||||
// Once the hold is gone the held entry goes too
|
||||
let purged = entries::purge(store, later, |_| false).await.unwrap();
|
||||
assert!(purged.removed >= 1, "{purged:?}");
|
||||
assert!(entry_for(test, "Internal with Bcc").await.is_none());
|
||||
assert!(
|
||||
entries::verify(store, Some(blobs))
|
||||
.await
|
||||
.unwrap()
|
||||
.iter()
|
||||
.all(|r| r.broken_at.is_none())
|
||||
);
|
||||
|
||||
// Changing a journal's retention doesn't touch what it has taken
|
||||
let before = entry_for(test, "Leaving").await.unwrap().1.expires_at;
|
||||
let (_, response) = call(
|
||||
&admin,
|
||||
"inbuxa:Journal/set",
|
||||
json!({"update": {outgoing.clone(): {"retentionDays": 30}}}),
|
||||
)
|
||||
.await;
|
||||
assert!(response["updated"].get(&outgoing).is_some(), "{response}");
|
||||
assert_eq!(
|
||||
entry_for(test, "Leaving").await.unwrap().1.expires_at,
|
||||
before
|
||||
);
|
||||
|
||||
// Journals turned off or removed take nothing more; entries stay
|
||||
let (_, response) = call(
|
||||
&admin,
|
||||
"inbuxa:Journal/set",
|
||||
json!({"update": {everything.clone(): {"enabled": false}}, "destroy": [outgoing]}),
|
||||
)
|
||||
.await;
|
||||
assert!(response["updated"].get(&everything).is_some(), "{response}");
|
||||
assert_eq!(response["destroyed"].as_array().map(|d| d.len()), Some(1));
|
||||
inbuxa_features::journal::invalidate();
|
||||
let count = all_entries(test).await.len();
|
||||
let response = send(
|
||||
&sender,
|
||||
&identity,
|
||||
&mailbox,
|
||||
&["[email protected]"],
|
||||
&["[email protected]"],
|
||||
"After the journals",
|
||||
)
|
||||
.await;
|
||||
assert!(response["created"].get("s").is_some(), "{response}");
|
||||
assert_eq!(all_entries(test).await.len(), count);
|
||||
assert!(entry_for(test, "Leaving").await.is_some());
|
||||
|
||||
// Every change to a journal is in the audit log
|
||||
let (_, response) = call(
|
||||
&admin,
|
||||
"inbuxa:AuditEvent/query",
|
||||
json!({"filter": {"targetKind": "inbuxa:Journal"}}),
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
response["ids"].as_array().map_or(0, |ids| ids.len()) >= 4,
|
||||
"{response}"
|
||||
);
|
||||
}
|
||||
|
||||
struct Raw(Vec<u8>);
|
||||
|
||||
impl Deserialize for Raw {
|
||||
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
|
||||
Ok(Raw(bytes.to_vec()))
|
||||
}
|
||||
}
|
||||
|
||||
#[ignore]
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
pub async fn journal_tests() {
|
||||
let mut test = TestServerBuilder::new("journal_tests")
|
||||
.await
|
||||
.with_default_listeners()
|
||||
.await
|
||||
.build()
|
||||
.await;
|
||||
let admin = test.create_admin_account("[email protected]").await;
|
||||
test.insert_account(admin);
|
||||
self::test(&mut test).await;
|
||||
if test.is_reset() {
|
||||
test.temp_dir.delete();
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -14,6 +14,9 @@ pub mod ai_explain;
|
||||
pub mod account_lock; // inbuxa: account lock with delegation
|
||||
pub mod legal_hold; // inbuxa: legal hold
|
||||
pub mod compliance; // inbuxa: the compliance roles
|
||||
pub mod mail_rules; // inbuxa: DLP and mail flow rules
|
||||
pub mod security_acceptances; // inbuxa: accepted security to-do items
|
||||
pub mod journal; // inbuxa: journaling
|
||||
pub mod audit; // inbuxa: the audit log
|
||||
pub mod authorization;
|
||||
pub mod auto_reload; // inbuxa: registry writes apply at once
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! `inbuxa:SecurityAcceptance` (security to-do list spec, SS-23 to SS-26):
|
||||
//! an accepted item is kept with who, when and why, a note is required,
|
||||
//! nothing is edited, only administrators may accept, and every acceptance
|
||||
//! made or removed is in the audit log.
|
||||
|
||||
use crate::utils::{
|
||||
account::Account,
|
||||
server::{TestServer, TestServerBuilder},
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
const USING: &[&str] = &[
|
||||
"urn:ietf:params:jmap:core",
|
||||
"urn:inbuxa:jmap",
|
||||
"urn:inbuxa:jmap:registry",
|
||||
];
|
||||
|
||||
async fn call(account: &Account, method: &str, mut arguments: Value) -> (String, Value) {
|
||||
if arguments.get("accountId").is_none() {
|
||||
arguments["accountId"] = account.id_string().into();
|
||||
}
|
||||
let response = account
|
||||
.jmap_request(USING, json!([[method, arguments, "0"]]))
|
||||
.await;
|
||||
let call = response
|
||||
.0
|
||||
.pointer("/methodResponses/0")
|
||||
.cloned()
|
||||
.unwrap_or_else(|| panic!("{method}: {}", response.0));
|
||||
(
|
||||
call[0].as_str().unwrap_or_default().to_string(),
|
||||
call[1].clone(),
|
||||
)
|
||||
}
|
||||
|
||||
async fn list(account: &Account) -> Vec<Value> {
|
||||
let (name, response) = call(
|
||||
account,
|
||||
"inbuxa:SecurityAcceptance/get",
|
||||
json!({"ids": null}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(name, "inbuxa:SecurityAcceptance/get", "{response}");
|
||||
response["list"].as_array().unwrap().clone()
|
||||
}
|
||||
|
||||
pub async fn test(test: &mut TestServer) {
|
||||
println!("Running security acceptance tests...");
|
||||
let admin = test.account("[email protected]");
|
||||
|
||||
// Accepted, with the server's who and when
|
||||
let (_, response) = call(
|
||||
&admin,
|
||||
"inbuxa:SecurityAcceptance/set",
|
||||
json!({"create": {
|
||||
"plain": {
|
||||
"check": "SS-1",
|
||||
"subject": "",
|
||||
"acceptedValue": true,
|
||||
"note": " Old scanners on the LAN; replaced in March. "
|
||||
},
|
||||
"relay": {
|
||||
"check": "SS-2",
|
||||
"acceptedValue": {"match": {}, "else": "is_local_ip(remote_ip)"},
|
||||
"note": "The office printer relays through us."
|
||||
}
|
||||
}}),
|
||||
)
|
||||
.await;
|
||||
let plain_id = response["created"]["plain"]["id"]
|
||||
.as_str()
|
||||
.unwrap_or_else(|| panic!("accepted: {response}"))
|
||||
.to_string();
|
||||
assert_eq!(
|
||||
response["created"]["plain"]["acceptedBy"], "[email protected]",
|
||||
"{response}"
|
||||
);
|
||||
let relay_id = response["created"]["relay"]["id"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.to_string();
|
||||
|
||||
let all = list(&admin).await;
|
||||
assert_eq!(all.len(), 2, "{all:?}");
|
||||
let plain = all.iter().find(|a| a["id"] == plain_id.as_str()).unwrap();
|
||||
assert_eq!(plain["check"], "SS-1");
|
||||
assert_eq!(plain["subject"], "");
|
||||
assert_eq!(plain["acceptedValue"], true);
|
||||
assert_eq!(plain["note"], "Old scanners on the LAN; replaced in March.");
|
||||
assert!(
|
||||
plain["acceptedAt"]
|
||||
.as_str()
|
||||
.is_some_and(|d| d.ends_with('Z')),
|
||||
"{plain}"
|
||||
);
|
||||
let relay = all.iter().find(|a| a["id"] == relay_id.as_str()).unwrap();
|
||||
assert_eq!(relay["acceptedValue"]["else"], "is_local_ip(remote_ip)");
|
||||
|
||||
// A note is required, the check must be one of ours, and what the
|
||||
// server sets can't be sent
|
||||
let (_, response) = call(
|
||||
&admin,
|
||||
"inbuxa:SecurityAcceptance/set",
|
||||
json!({"create": {
|
||||
"nonote": {"check": "SS-1", "acceptedValue": true, "note": " "},
|
||||
"nocheck": {"check": "SS-99", "acceptedValue": true, "note": "x"},
|
||||
"by": {"check": "SS-1", "acceptedValue": true, "note": "x", "acceptedBy": "someone"}
|
||||
}}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
response["notCreated"]["nonote"]["properties"][0], "note",
|
||||
"{response}"
|
||||
);
|
||||
assert_eq!(
|
||||
response["notCreated"]["nocheck"]["properties"][0], "check",
|
||||
"{response}"
|
||||
);
|
||||
assert_eq!(
|
||||
response["notCreated"]["by"]["properties"][0], "acceptedBy",
|
||||
"{response}"
|
||||
);
|
||||
assert_eq!(list(&admin).await.len(), 2);
|
||||
|
||||
// Replaced, never edited
|
||||
let (name, response) = call(
|
||||
&admin,
|
||||
"inbuxa:SecurityAcceptance/set",
|
||||
json!({"update": {plain_id.as_str(): {"note": "changed"}}}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(name, "error", "an acceptance was edited: {response}");
|
||||
|
||||
// Only administrators: someone without the permissions neither sees
|
||||
// nor accepts
|
||||
let user = admin
|
||||
.create_user_account(
|
||||
"[email protected]",
|
||||
"user-secret-8812",
|
||||
"User",
|
||||
&[],
|
||||
vec![],
|
||||
)
|
||||
.await;
|
||||
let (name, response) = call(
|
||||
&user,
|
||||
"inbuxa:SecurityAcceptance/set",
|
||||
json!({"create": {"x": {"check": "SS-1", "acceptedValue": true, "note": "mine"}}}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(name, "error", "a user accepted an item: {response}");
|
||||
let (name, response) = call(&user, "inbuxa:SecurityAcceptance/get", json!({"ids": null})).await;
|
||||
assert_eq!(name, "error", "a user read acceptances: {response}");
|
||||
|
||||
// Removed
|
||||
let (_, response) = call(
|
||||
&admin,
|
||||
"inbuxa:SecurityAcceptance/set",
|
||||
json!({"destroy": [plain_id, "zzzzzz"]}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response["destroyed"], json!([plain_id]), "{response}");
|
||||
assert!(
|
||||
response["notDestroyed"].get("zzzzzz").is_some(),
|
||||
"{response}"
|
||||
);
|
||||
let remaining = list(&admin).await;
|
||||
assert_eq!(remaining.len(), 1);
|
||||
assert_eq!(remaining[0]["check"], "SS-2");
|
||||
|
||||
// SS-26: accepted and removed are both in the audit log, with who and
|
||||
// what
|
||||
let (_, response) = call(
|
||||
&admin,
|
||||
"inbuxa:AuditEvent/query",
|
||||
json!({"filter": {"targetKind": "inbuxa:SecurityAcceptance"}}),
|
||||
)
|
||||
.await;
|
||||
let ids = response["ids"].clone();
|
||||
let (_, response) = call(&admin, "inbuxa:AuditEvent/get", json!({"ids": ids})).await;
|
||||
let events = response["list"].as_array().unwrap();
|
||||
let created = events
|
||||
.iter()
|
||||
.filter(|e| e["action"] == "create" && e["outcome"]["status"] == "success")
|
||||
.count();
|
||||
assert_eq!(created, 2, "{response}");
|
||||
let removed = events
|
||||
.iter()
|
||||
.find(|e| e["action"] == "destroy" && e["outcome"]["status"] == "success")
|
||||
.unwrap_or_else(|| panic!("no removal recorded: {response}"));
|
||||
assert_eq!(removed["target"]["name"], "SS-1", "{removed}");
|
||||
assert!(
|
||||
events
|
||||
.iter()
|
||||
.all(|e| e["actor"]["name"] == "[email protected]"),
|
||||
"{response}"
|
||||
);
|
||||
assert!(
|
||||
response.to_string().contains("Old scanners on the LAN"),
|
||||
"the note isn't in the record: {response}"
|
||||
);
|
||||
|
||||
// Cleared for the tests that follow
|
||||
call(
|
||||
&admin,
|
||||
"inbuxa:SecurityAcceptance/set",
|
||||
json!({"destroy": [relay_id]}),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[ignore]
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
pub async fn security_acceptance_tests() {
|
||||
let mut test = TestServerBuilder::new("security_acceptance_tests")
|
||||
.await
|
||||
.with_default_listeners()
|
||||
.await
|
||||
.build()
|
||||
.await;
|
||||
let admin = test.create_admin_account("[email protected]").await;
|
||||
test.insert_account(admin);
|
||||
self::test(&mut test).await;
|
||||
if test.is_reset() {
|
||||
test.temp_dir.delete();
|
||||
}
|
||||
}
|
||||
@@ -112,3 +112,18 @@ a fresh copy for each one. See `docs/spec/compat-tests.md`.
|
||||
tools/fork/run-compat.sh --store /srv/inbuxa-copy/rocks.db \
|
||||
--admin 'admin@example.org:PASSWORD' --recordings ~/compat
|
||||
```
|
||||
|
||||
## expr-schema.py
|
||||
|
||||
Writes each expression field's allowed constants and variables, read from the
|
||||
generated registry code, into the schema the server serves INBUXA Admin
|
||||
(`resources/schema/schema.json.gz` and its checksum). The admin uses them to
|
||||
offer plain choices instead of a free-text box.
|
||||
|
||||
```bash
|
||||
tools/fork/expr-schema.py # update the schema
|
||||
tools/fork/expr-schema.py --check # exit 1 if it's out of date (CI)
|
||||
```
|
||||
|
||||
Re-run it after anything that regenerates the registry, an upstream import
|
||||
included.
|
||||
|
||||
Executable
+144
@@ -0,0 +1,144 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Tell INBUXA Admin what each expression field accepts.
|
||||
|
||||
Every expression field in the registry has a context: the constants it may
|
||||
evaluate to (DKIM verification: relaxed, strict or disable) and the variables
|
||||
its conditions may read (sender_domain, local_port...). The server enforces
|
||||
both, but the schema it serves the admin describes every expression field as
|
||||
only an `x:Expression` object, so the admin can offer nothing better than a
|
||||
free-text box.
|
||||
|
||||
This reads those contexts from the generated registry code and writes them
|
||||
into the served schema, on each expression field's type:
|
||||
|
||||
"type": {"type": "object", "objectName": "x:Expression",
|
||||
"expression": {"constants": ["relaxed", "strict", "disable"],
|
||||
"variables": ["sender", "sender_domain", ...]}}
|
||||
|
||||
The registry code is the source, so re-run this after anything that
|
||||
regenerates it (an upstream import, a new expression field). `--check` exits 1
|
||||
when the schema is out of date; CI runs it.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
REGISTRY = root / 'crates' / 'registry' / 'src' / 'schema'
|
||||
SCHEMA = root / 'resources' / 'schema' / 'schema.json.gz'
|
||||
SCHEMA_HASH = root / 'resources' / 'schema' / 'schema.json.sha256'
|
||||
|
||||
|
||||
def names(enum, text):
|
||||
"""Variant → wire name, from the `Enum::Variant => "name"` arms."""
|
||||
return dict(re.findall(rf'{enum}::(\w+) => "([^"]+)"', text))
|
||||
|
||||
|
||||
def lists(text):
|
||||
"""Every `pub static NAME: &[ExpressionConstant|Variable] = &[...]`."""
|
||||
out = {}
|
||||
for name, kind, body in re.findall(
|
||||
r'pub static (\w+): &\[(ExpressionConstant|ExpressionVariable)\] = &\[(.*?)\];', text, re.S
|
||||
):
|
||||
out[name] = (kind, re.findall(rf'{kind}::(\w+)', body))
|
||||
return out
|
||||
|
||||
|
||||
def contexts(text):
|
||||
"""(struct, Property variant, variables list name, constants list name) per context."""
|
||||
out = []
|
||||
for block in re.finditer(r'(?m)^impl (\w+) \{(.*?)^\}', text, re.S):
|
||||
struct, body = block.group(1), block.group(2)
|
||||
for ctx in re.finditer(r'ExpressionContext \{(.*?)\n\s*\}\n', body, re.S):
|
||||
fields = ctx.group(1)
|
||||
prop = re.search(r'property: Property::(\w+),', fields)
|
||||
var = re.search(r'allowed_variables: (&\[\]|\w+),', fields)
|
||||
const = re.search(r'allowed_constants: (&\[\]|\w+),', fields)
|
||||
if prop and var and const:
|
||||
out.append((struct, prop.group(1), var.group(1), const.group(1)))
|
||||
return out
|
||||
|
||||
|
||||
def build():
|
||||
enums = (REGISTRY / 'enums.rs').read_text(encoding='utf-8')
|
||||
enums_impl = (REGISTRY / 'enums_impl.rs').read_text(encoding='utf-8')
|
||||
props = names('Property', (REGISTRY / 'properties_impl.rs').read_text(encoding='utf-8'))
|
||||
const_names = names('ExpressionConstant', enums_impl)
|
||||
var_names = names('ExpressionVariable', enums_impl)
|
||||
known = lists(enums)
|
||||
|
||||
def resolve(ref, kind, wire):
|
||||
if ref == '&[]':
|
||||
return []
|
||||
found = known.get(ref)
|
||||
if not found or found[0] != kind:
|
||||
raise SystemExit(f'expr-schema: no {kind} list named {ref}')
|
||||
return [wire[v] for v in found[1]]
|
||||
|
||||
table = {}
|
||||
for struct, prop, var, const in contexts((REGISTRY / 'structs_impl.rs').read_text(encoding='utf-8')):
|
||||
table[(f'x:{struct}', props[prop])] = {
|
||||
'constants': resolve(const, 'ExpressionConstant', const_names),
|
||||
'variables': resolve(var, 'ExpressionVariable', var_names),
|
||||
}
|
||||
return table
|
||||
|
||||
|
||||
def apply(schema, table):
|
||||
"""Write the table into the schema; returns the (object, field) pairs it couldn't place."""
|
||||
missing = []
|
||||
for (obj, field), expr in sorted(table.items()):
|
||||
target = schema['fields'].get(obj, {}).get('properties', {}).get(field)
|
||||
if target is None or target['type'].get('objectName') != 'x:Expression':
|
||||
missing.append(f'{obj}.{field}')
|
||||
continue
|
||||
target['type']['expression'] = expr
|
||||
return missing
|
||||
|
||||
|
||||
def encode(schema):
|
||||
text = json.dumps(schema, ensure_ascii=False, separators=(',', ':'))
|
||||
out = gzip.compress(text.encode('utf-8'), compresslevel=9, mtime=0)
|
||||
digest = base64.urlsafe_b64encode(hashlib.sha256(out).digest()).decode().rstrip('=')
|
||||
return out, digest
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument('--check', action='store_true', help='exit 1 if the schema is out of date')
|
||||
args = parser.parse_args()
|
||||
|
||||
before = SCHEMA.read_bytes()
|
||||
schema = json.loads(gzip.decompress(before))
|
||||
table = build()
|
||||
missing = apply(schema, table)
|
||||
if missing:
|
||||
print('expr-schema: expression contexts with no matching schema field:', file=sys.stderr)
|
||||
for m in missing:
|
||||
print(f' {m}', file=sys.stderr)
|
||||
return 1
|
||||
|
||||
current = json.loads(gzip.decompress(before))
|
||||
if current == schema:
|
||||
print(f'expr-schema: {len(table)} expression fields, schema up to date')
|
||||
return 0
|
||||
if args.check:
|
||||
print('expr-schema: schema is out of date; run tools/fork/expr-schema.py', file=sys.stderr)
|
||||
return 1
|
||||
out, digest = encode(schema)
|
||||
SCHEMA.write_bytes(out)
|
||||
SCHEMA_HASH.write_text(digest, encoding='utf-8')
|
||||
print(f'expr-schema: wrote {len(table)} expression fields into {SCHEMA.relative_to(root)}')
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user