Journaling: capture at the queue, the built-in journal, retention
Phase 2 of the journaling spec. - A copy of each message is taken in MessageWrapper::queue, after DLP and transport rules, for every enabled journal that takes it (direction and scope: everyone, or accounts, groups, domains, tenants). If the copy can't be taken the message isn't queued (temporary failure). - The journal report: the envelope one field a line (sender, To, Cc, Bcc from the envelope, list members from their ORCPT, direction, held for review), then the queued message byte for byte as message/rfc822. - The built-in journal under J in the inbuxa subspace: one chain per node whose links name each entry by SHA-256, so entries can expire out of chain order; purge leaves a marker, and verify catches an entry changed or removed early and a report that doesn't match. - Retention per journal (30 to 3650 days); an entry keeps what it was written with. The daily maintenance purges what's due, keeping entries whose people a legal hold covers (deleted accounts a hold keeps too), and records the counts in the audit log. - inbuxa:Journal get/set, audited by the request layer. Permissions 680-683: administrators see and change journals; the Compliance Officer sees, searches and exports. Whoever changes journals may grant search and export without holding them, so officers can still be appointed. - Catalog entries (inbuxa:Journal, source "journal"); spec as-built notes. tests/src/system/journal.rs: validation, internal mail with a Bcc, outgoing into two journals, incoming over LMTP, the report and its original, tamper and early removal caught, hold-aware purge, retention changes leave entries alone, disabled and removed journals take nothing.
This commit is contained in:
@@ -0,0 +1,359 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! The journal report (JR-3, JR-4): a message whose first part lists the
|
||||
//! envelope, one field a line, and whose second part is the message as it
|
||||
//! was queued, byte for byte, as `message/rfc822`. Field names are fixed
|
||||
//! English: a report is a record, and scripts read it.
|
||||
|
||||
use super::Direction;
|
||||
use mail_builder::headers::{Header, date::Date, text::Text};
|
||||
use mail_parser::MessageParser;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
/// One envelope recipient, with the address it was given as (a list's, for
|
||||
/// the list's members).
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Recipient {
|
||||
pub address: String,
|
||||
pub orcpt: Option<String>,
|
||||
}
|
||||
|
||||
/// What the queue knows about a message.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Envelope<'x> {
|
||||
pub sender: &'x str,
|
||||
pub authenticated: bool,
|
||||
pub recipients: &'x [Recipient],
|
||||
pub queue_id: u64,
|
||||
/// Seconds.
|
||||
pub received: u64,
|
||||
pub direction: Direction,
|
||||
pub held: bool,
|
||||
}
|
||||
|
||||
/// What a report says, besides the envelope's own fields.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct Fields {
|
||||
pub subject: String,
|
||||
pub message_id: String,
|
||||
pub to: Vec<String>,
|
||||
pub cc: Vec<String>,
|
||||
/// Envelope recipients in neither To nor Cc, nor reached through a list.
|
||||
pub bcc: Vec<String>,
|
||||
/// A list's address, and its members among the recipients.
|
||||
pub expanded: Vec<(String, Vec<String>)>,
|
||||
}
|
||||
|
||||
/// One line's worth of a value: no line breaks, no control characters.
|
||||
fn line(value: &str) -> String {
|
||||
value
|
||||
.chars()
|
||||
.map(|c| if c.is_control() { ' ' } else { c })
|
||||
.collect::<String>()
|
||||
.trim()
|
||||
.to_string()
|
||||
}
|
||||
|
||||
/// The address an ORCPT names, without its `rfc822;` type.
|
||||
fn orcpt_address(orcpt: &str) -> String {
|
||||
let orcpt = orcpt.trim();
|
||||
let bare = match orcpt.split_once(';') {
|
||||
Some((kind, address)) if kind.eq_ignore_ascii_case("rfc822") => address,
|
||||
_ => orcpt,
|
||||
};
|
||||
bare.trim().to_lowercase()
|
||||
}
|
||||
|
||||
/// Sorts the envelope's recipients by how they were addressed.
|
||||
pub fn fields(envelope: &Envelope<'_>, original: &[u8]) -> Fields {
|
||||
let parsed = MessageParser::default().parse_headers(original);
|
||||
let headed = |which: Option<&mail_parser::Address<'_>>| -> Vec<String> {
|
||||
which
|
||||
.map(|list| {
|
||||
list.iter()
|
||||
.filter_map(|addr| addr.address())
|
||||
.map(|address| address.to_lowercase())
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default()
|
||||
};
|
||||
let (subject, message_id, header_to, header_cc) = match &parsed {
|
||||
Some(message) => (
|
||||
message.subject().map(line).unwrap_or_default(),
|
||||
message
|
||||
.message_id()
|
||||
.map(|id| format!("<{}>", line(id)))
|
||||
.unwrap_or_default(),
|
||||
headed(message.to()),
|
||||
headed(message.cc()),
|
||||
),
|
||||
None => Default::default(),
|
||||
};
|
||||
|
||||
let mut fields = Fields {
|
||||
subject,
|
||||
message_id,
|
||||
..Default::default()
|
||||
};
|
||||
for rcpt in envelope.recipients {
|
||||
let address = rcpt.address.to_lowercase();
|
||||
let via = rcpt
|
||||
.orcpt
|
||||
.as_deref()
|
||||
.map(orcpt_address)
|
||||
.filter(|via| !via.is_empty() && *via != address);
|
||||
if header_to.contains(&address) {
|
||||
fields.to.push(line(&rcpt.address));
|
||||
} else if header_cc.contains(&address) {
|
||||
fields.cc.push(line(&rcpt.address));
|
||||
} else if let Some(via) = via {
|
||||
match fields.expanded.iter_mut().find(|(list, _)| *list == via) {
|
||||
Some((_, members)) => members.push(line(&rcpt.address)),
|
||||
None => fields
|
||||
.expanded
|
||||
.push((line(&via), vec![line(&rcpt.address)])),
|
||||
}
|
||||
} else {
|
||||
fields.bcc.push(line(&rcpt.address));
|
||||
}
|
||||
}
|
||||
fields
|
||||
}
|
||||
|
||||
/// The report's first part.
|
||||
pub fn text(envelope: &Envelope<'_>, fields: &Fields) -> String {
|
||||
let mut out = String::new();
|
||||
let mut field = |name: &str, value: &str| {
|
||||
if !value.is_empty() {
|
||||
out.push_str(name);
|
||||
out.push_str(": ");
|
||||
out.push_str(value);
|
||||
out.push_str("\r\n");
|
||||
}
|
||||
};
|
||||
let sender = if envelope.sender.is_empty() {
|
||||
"<>".to_string()
|
||||
} else {
|
||||
line(envelope.sender)
|
||||
};
|
||||
field("Sender", &sender);
|
||||
field(
|
||||
"Authenticated",
|
||||
if envelope.authenticated { "yes" } else { "no" },
|
||||
);
|
||||
field("Subject", &fields.subject);
|
||||
field("Message-ID", &fields.message_id);
|
||||
field("Queue ID", &format!("{:x}", envelope.queue_id));
|
||||
field(
|
||||
"Received",
|
||||
&mail_parser::DateTime::from_timestamp(envelope.received as i64).to_rfc3339(),
|
||||
);
|
||||
field("Direction", envelope.direction.as_str());
|
||||
field("To", &fields.to.join(", "));
|
||||
field("Cc", &fields.cc.join(", "));
|
||||
field("Bcc", &fields.bcc.join(", "));
|
||||
for (list, members) in &fields.expanded {
|
||||
field("Expanded", &format!("{list} -> {}", members.join(", ")));
|
||||
}
|
||||
if envelope.held {
|
||||
field("Held for review", "yes");
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn hex(bytes: &[u8]) -> String {
|
||||
bytes.iter().map(|b| format!("{b:02x}")).collect()
|
||||
}
|
||||
|
||||
/// Whether a message can travel as 8bit: no NULs, no line past 998 bytes.
|
||||
fn fits_8bit(message: &[u8]) -> bool {
|
||||
!message.contains(&0) && message.split(|b| *b == b'\n').all(|l| l.len() <= 998)
|
||||
}
|
||||
|
||||
/// The whole report: headers, the fields, then the original untouched.
|
||||
/// `from` is the address the report is from; `host` names the server in its
|
||||
/// Message-ID.
|
||||
pub fn build(
|
||||
envelope: &Envelope<'_>,
|
||||
original: &[u8],
|
||||
from: &str,
|
||||
host: &str,
|
||||
) -> (Vec<u8>, Fields) {
|
||||
let fields = fields(envelope, original);
|
||||
let body = text(envelope, &fields);
|
||||
// A boundary that can't occur in the original
|
||||
let mut boundary = format!("journal-{}", &hex(&Sha256::digest(original))[..32]);
|
||||
while original
|
||||
.windows(boundary.len())
|
||||
.any(|window| window == boundary.as_bytes())
|
||||
{
|
||||
boundary.push('x');
|
||||
}
|
||||
|
||||
let mut out: Vec<u8> = Vec::with_capacity(original.len() + body.len() + 1024);
|
||||
out.extend_from_slice(format!("From: Journal <{}>\r\n", line(from)).as_bytes());
|
||||
out.extend_from_slice(b"Date: ");
|
||||
out.extend_from_slice(Date::new(envelope.received as i64).to_rfc822().as_bytes());
|
||||
out.extend_from_slice(b"\r\n");
|
||||
out.extend_from_slice(b"Subject: ");
|
||||
let subject = if fields.subject.is_empty() {
|
||||
"Journal report".to_string()
|
||||
} else {
|
||||
format!("Journal report: {}", fields.subject)
|
||||
};
|
||||
Text::new(subject).write_header(&mut out, "Subject: ".len());
|
||||
out.extend_from_slice(
|
||||
format!(
|
||||
"Message-ID: <journal.{:x}.{}@{}>\r\n",
|
||||
envelope.queue_id,
|
||||
envelope.received,
|
||||
line(host)
|
||||
)
|
||||
.as_bytes(),
|
||||
);
|
||||
out.extend_from_slice(format!("X-Inbuxa-Journal: {:x}\r\n", envelope.queue_id).as_bytes());
|
||||
out.extend_from_slice(b"MIME-Version: 1.0\r\n");
|
||||
out.extend_from_slice(
|
||||
format!("Content-Type: multipart/mixed; boundary=\"{boundary}\"\r\n\r\n").as_bytes(),
|
||||
);
|
||||
out.extend_from_slice(format!("--{boundary}\r\n").as_bytes());
|
||||
out.extend_from_slice(
|
||||
b"Content-Type: text/plain; charset=utf-8\r\nContent-Transfer-Encoding: 8bit\r\n\r\n",
|
||||
);
|
||||
out.extend_from_slice(body.as_bytes());
|
||||
out.extend_from_slice(format!("\r\n--{boundary}\r\n").as_bytes());
|
||||
out.extend_from_slice(b"Content-Type: message/rfc822\r\n");
|
||||
out.extend_from_slice(b"Content-Disposition: attachment; filename=\"original.eml\"\r\n");
|
||||
out.extend_from_slice(if fits_8bit(original) {
|
||||
b"Content-Transfer-Encoding: 8bit\r\n\r\n".as_slice()
|
||||
} else {
|
||||
b"Content-Transfer-Encoding: binary\r\n\r\n".as_slice()
|
||||
});
|
||||
out.extend_from_slice(original);
|
||||
// The line break before a boundary belongs to the boundary: the
|
||||
// original keeps its own last one
|
||||
out.extend_from_slice(format!("\r\n--{boundary}--\r\n").as_bytes());
|
||||
(out, fields)
|
||||
}
|
||||
|
||||
/// Where the original starts and ends inside a report [`build`] made.
|
||||
pub fn original(report: &[u8]) -> Option<&[u8]> {
|
||||
let parsed = MessageParser::default().parse(report)?;
|
||||
let part = parsed.attachment(0)?;
|
||||
let start = part.raw_body_offset() as usize;
|
||||
let end = part.raw_end_offset() as usize;
|
||||
report.get(start..end)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const ORIGINAL: &[u8] = b"From: [email protected]\r\n\
|
||||
To: Bank <[email protected]>\r\n\
|
||||
Cc: [email protected]\r\n\
|
||||
Subject: Q3 figures\r\n\
|
||||
Message-ID: <[email protected]>\r\n\
|
||||
\r\n\
|
||||
The figures.\r\n";
|
||||
|
||||
fn rcpt(address: &str, orcpt: Option<&str>) -> Recipient {
|
||||
Recipient {
|
||||
address: address.into(),
|
||||
orcpt: orcpt.map(Into::into),
|
||||
}
|
||||
}
|
||||
|
||||
fn envelope(recipients: &[Recipient]) -> Envelope<'_> {
|
||||
Envelope {
|
||||
sender: "[email protected]",
|
||||
authenticated: true,
|
||||
recipients,
|
||||
queue_id: 0x1a2b,
|
||||
received: 1_790_000_000,
|
||||
direction: Direction::Outgoing,
|
||||
held: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recipients_sorted_by_how_they_were_addressed() {
|
||||
let recipients = [
|
||||
rcpt("[email protected]", None),
|
||||
rcpt("[email protected]", Some("rfc822;[email protected]")),
|
||||
rcpt("[email protected]", None),
|
||||
rcpt("[email protected]", Some("[email protected]")),
|
||||
rcpt("[email protected]", Some("rfc822;[email protected]")),
|
||||
];
|
||||
let fields = fields(&envelope(&recipients), ORIGINAL);
|
||||
assert_eq!(fields.subject, "Q3 figures");
|
||||
assert_eq!(fields.message_id, "<[email protected]>");
|
||||
assert_eq!(fields.to, vec!["[email protected]"]);
|
||||
assert_eq!(fields.cc, vec!["[email protected]"]);
|
||||
assert_eq!(fields.bcc, vec!["[email protected]"]);
|
||||
assert_eq!(
|
||||
fields.expanded,
|
||||
vec![(
|
||||
"[email protected]".to_string(),
|
||||
vec![
|
||||
"[email protected]".to_string(),
|
||||
"[email protected]".to_string()
|
||||
]
|
||||
)]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn report_carries_the_original_untouched() {
|
||||
let recipients = [
|
||||
rcpt("[email protected]", None),
|
||||
rcpt("[email protected]", None),
|
||||
];
|
||||
let (report, _) = build(
|
||||
&envelope(&recipients),
|
||||
ORIGINAL,
|
||||
"[email protected]",
|
||||
"mx.example.com",
|
||||
);
|
||||
let text = String::from_utf8_lossy(&report);
|
||||
assert!(text.contains("Sender: [email protected]\r\n"));
|
||||
assert!(text.contains("Bcc: [email protected]\r\n"));
|
||||
assert!(text.contains("Queue ID: 1a2b\r\n"));
|
||||
assert!(text.contains("Direction: outgoing\r\n"));
|
||||
assert!(text.contains("Subject: Journal report: Q3 figures\r\n"));
|
||||
assert!(!text.contains("Held for review"));
|
||||
assert_eq!(original(&report), Some(ORIGINAL));
|
||||
let unterminated = &ORIGINAL[..ORIGINAL.len() - 2];
|
||||
let (report, _) = build(
|
||||
&envelope(&recipients),
|
||||
unterminated,
|
||||
"[email protected]",
|
||||
"mx.example.com",
|
||||
);
|
||||
assert_eq!(original(&report), Some(unterminated));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn values_stay_on_one_line() {
|
||||
let recipients = [rcpt("[email protected]", None)];
|
||||
let mut env = envelope(&recipients);
|
||||
env.sender = "[email protected]\r\nBcc: [email protected]";
|
||||
env.held = true;
|
||||
let body = text(&env, &Fields::default());
|
||||
assert_eq!(body.matches("\r\n").count(), body.lines().count());
|
||||
assert!(body.contains("Sender: [email protected] Bcc: [email protected]\r\n"));
|
||||
assert!(body.contains("Held for review: yes\r\n"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_sender_is_shown_as_such() {
|
||||
let recipients = [rcpt("[email protected]", None)];
|
||||
let mut env = envelope(&recipients);
|
||||
env.sender = "";
|
||||
assert!(text(&env, &Fields::default()).starts_with("Sender: <>\r\n"));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user