inbuxa/inbuxa-server
1
1 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 54 Wiki Activity Sponsor

Journal reports: move to a documented multipart/report #287

Merged
jcoffey-dev merged 1 commits from feat/journal-report-format into main 2026-11-04 04:24:51 +00:00
Conversation 0 Commits 1 Files Changed 5
+607 -115
5 changed files with 607 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

crates/features/src/journal/report.rs
+416 -91
View File
@@ -4,16 +4,35 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! The journal report (JR-3, JR-4): a message whose first part lists the
//! envelope, one field a line, and whose second part is the message as it
//! was queued, byte for byte, as `message/rfc822`. Field names are fixed
//! English: a report is a record, and scripts read it.
//! The journal report (JR-3, JR-4), as coffeylabs.org/specs/journal-report/
//! defines it: a `multipart/report` whose first part is a summary for people,
//! whose second lists the envelope for programs, and whose third is the
//! message as it was queued, byte for byte, as `message/rfc822`.
//!
//! Reports already stored keep the layout they were written in, a
//! `multipart/mixed` of the summary and the message, and [`original`] reads
//! both.
use super::Direction;
use mail_builder::headers::{Header, date::Date, text::Text};
use mail_parser::MessageParser;
use mail_parser::{MessageParser, MimeHeaders};
use sha2::{Digest, Sha256};
/// The `report-type` of the `multipart/report`: RFC 6522 has it be the
/// subtype of the second part.
pub const REPORT_TYPE: &str = "vnd.coffeylabs.journal-report";
/// The media type of the part programs read.
pub const MEDIA_TYPE: &str = "application/vnd.coffeylabs.journal-report";
/// The version that part declares.
pub const VERSION: u32 = 1;
/// The longest value the summary shows: a line of a message that travels as
/// 8bit can't pass 998 bytes, and the original holds the whole of it.
const SUMMARY_VALUE_MAX: usize = 900;
/// The longest address taken from a sender's ORCPT parameter (RFC 5321
/// allows a path of 256; this leaves room for internationalized ones).
const VIA_MAX: usize = 320;
/// One envelope recipient, with the address it was given as (a list's, for
/// the list's members).
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -37,19 +56,27 @@ pub struct Envelope<'x> {
pub held: bool,
}
/// How a recipient came to be on the envelope, as far as the message and
/// the envelope show.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Addressed {
To,
Cc,
/// Reached through another address, such as a list's: the ORCPT named it.
List(String),
/// A mail flow rule added it or redirected to it.
Rule(String),
/// In neither To nor Cc, with nothing to say why. A blind copy looks like
/// this, and so does a list's post, or a message arriving for an alias.
NotInHeaders,
}
/// What a report says, besides the envelope's own fields.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Fields {
pub subject: String,
pub message_id: String,
pub to: Vec<String>,
pub cc: Vec<String>,
/// Envelope recipients in neither To nor Cc, nor reached through a list.
pub bcc: Vec<String>,
/// A list's address, and its members among the recipients.
pub expanded: Vec<(String, Vec<String>)>,
/// A rule's name, and the recipients it added.
pub added: Vec<(String, Vec<String>)>,
pub recipients: Vec<(String, Addressed)>,
}
/// One line's worth of a value: no line breaks, no control characters.
@@ -62,14 +89,22 @@ fn line(value: &str) -> String {
.to_string()
}
/// The address an ORCPT names, without its `rfc822;` type.
fn orcpt_address(orcpt: &str) -> String {
/// The address an ORCPT names, without its `rfc822;` type. A sender writes
/// this parameter, so anything that isn't plainly an address is dropped: it
/// goes into a field programs read.
fn orcpt_address(orcpt: &str) -> Option<String> {
let orcpt = orcpt.trim();
let bare = match orcpt.split_once(';') {
Some((kind, address)) if kind.eq_ignore_ascii_case("rfc822") => address,
_ => orcpt,
};
bare.trim().to_lowercase()
let address = bare.trim().to_lowercase();
let plain = address.len() <= VIA_MAX
&& address.contains('@')
&& !address.chars().any(|c| {
c.is_whitespace() || c.is_control() || matches!(c, '<' | '>' | ';' | '"' | ',')
});
plain.then_some(address)
}
/// Sorts the envelope's recipients by how they were addressed.
@@ -98,49 +133,59 @@ pub fn fields(envelope: &Envelope<'_>, original: &[u8]) -> Fields {
None => Default::default(),
};
let mut fields = Fields {
let recipients = envelope
.recipients
.iter()
.map(|rcpt| {
let address = rcpt.address.to_lowercase();
let via = rcpt
.orcpt
.as_deref()
.and_then(orcpt_address)
.filter(|via| *via != address);
let addressed = if let Some(rule) = &rcpt.added_by {
Addressed::Rule(line(rule))
} else if header_to.contains(&address) {
Addressed::To
} else if header_cc.contains(&address) {
Addressed::Cc
} else if let Some(via) = via {
Addressed::List(via)
} else {
Addressed::NotInHeaders
};
(line(&rcpt.address), addressed)
})
.collect();
Fields {
subject,
message_id,
..Default::default()
};
for rcpt in envelope.recipients {
let address = rcpt.address.to_lowercase();
let via = rcpt
.orcpt
.as_deref()
.map(orcpt_address)
.filter(|via| !via.is_empty() && *via != address);
if let Some(rule) = &rcpt.added_by {
match fields.added.iter_mut().find(|(name, _)| name == rule) {
Some((_, added)) => added.push(line(&rcpt.address)),
None => fields.added.push((line(rule), vec![line(&rcpt.address)])),
}
} else if header_to.contains(&address) {
fields.to.push(line(&rcpt.address));
} else if header_cc.contains(&address) {
fields.cc.push(line(&rcpt.address));
} else if let Some(via) = via {
match fields.expanded.iter_mut().find(|(list, _)| *list == via) {
Some((_, members)) => members.push(line(&rcpt.address)),
None => fields
.expanded
.push((line(&via), vec![line(&rcpt.address)])),
}
} else {
fields.bcc.push(line(&rcpt.address));
}
recipients,
}
fields
}
/// The report's first part.
/// At most `max` bytes of a value, cut between characters.
fn fit(value: &str, max: usize) -> &str {
if value.len() <= max {
return value;
}
let mut end = max;
while !value.is_char_boundary(end) {
end -= 1;
}
&value[..end]
}
/// The report's first part, for people. One recipient a line, so a long
/// list can't make a line too long to travel. Programs read the second part.
pub fn text(envelope: &Envelope<'_>, fields: &Fields) -> String {
let mut out = String::new();
let mut field = |name: &str, value: &str| {
if !value.is_empty() {
out.push_str(name);
out.push_str(": ");
out.push_str(value);
out.push_str(fit(value, SUMMARY_VALUE_MAX));
out.push_str("\r\n");
}
};
@@ -162,14 +207,32 @@ pub fn text(envelope: &Envelope<'_>, fields: &Fields) -> String {
&mail_parser::DateTime::from_timestamp(envelope.received as i64).to_rfc3339(),
);
field("Direction", envelope.direction.as_str());
field("To", &fields.to.join(", "));
field("Cc", &fields.cc.join(", "));
field("Bcc", &fields.bcc.join(", "));
for (list, members) in &fields.expanded {
field("Expanded", &format!("{list} -> {}", members.join(", ")));
let of = |wanted: &dyn Fn(&Addressed) -> bool| -> Vec<&str> {
fields
.recipients
.iter()
.filter(|(_, addressed)| wanted(addressed))
.map(|(address, _)| address.as_str())
.collect()
};
for address in of(&|a| *a == Addressed::To) {
field("To", address);
}
for (rule, added) in &fields.added {
field("Added by rule", &format!("{rule} -> {}", added.join(", ")));
for address in of(&|a| *a == Addressed::Cc) {
field("Cc", address);
}
for address in of(&|a| *a == Addressed::NotInHeaders) {
field("Not in To or Cc", address);
}
for (address, addressed) in &fields.recipients {
if let Addressed::List(via) = addressed {
field("Expanded", &format!("{via} -> {address}"));
}
}
for (address, addressed) in &fields.recipients {
if let Addressed::Rule(rule) = addressed {
field("Added by rule", &format!("{rule} -> {address}"));
}
}
if envelope.held {
field("Held for review", "yes");
@@ -177,18 +240,100 @@ pub fn text(envelope: &Envelope<'_>, fields: &Fields) -> String {
out
}
/// A value of the second part's `rule` parameter.
fn quoted(value: &str) -> String {
let mut out = String::with_capacity(value.len() + 2);
out.push('"');
for c in value.chars() {
if matches!(c, '"' | '\\') {
out.push('\\');
}
out.push(c);
}
out.push('"');
out
}
/// The report's second part, for programs: one field a line, and one
/// `Original-Rcpt-To` for each envelope recipient. Field names are fixed
/// English, modeled on RFC 5965's where they mean the same.
pub fn machine(envelope: &Envelope<'_>, fields: &Fields, host: &str) -> String {
let mut out = format!(
"Version: {VERSION}\r\n\
Reporting-MTA: dns; {}\r\n\
Queue-ID: {:x}\r\n\
Arrival-Date: {}\r\n\
Direction: {}\r\n\
Authenticated: {}\r\n\
Held: {}\r\n\
Original-Mail-From: <{}>\r\n",
line(host),
envelope.queue_id,
Date::new(envelope.received as i64).to_rfc822(),
envelope.direction.as_str(),
if envelope.authenticated { "yes" } else { "no" },
if envelope.held { "yes" } else { "no" },
line(envelope.sender),
);
for (address, addressed) in &fields.recipients {
out.push_str("Original-Rcpt-To: <");
out.push_str(address);
out.push_str(">; addressed=");
match addressed {
Addressed::To => out.push_str("to"),
Addressed::Cc => out.push_str("cc"),
Addressed::List(via) => {
out.push_str("list; via=<");
out.push_str(via);
out.push('>');
}
Addressed::Rule(rule) => {
out.push_str("rule; rule=");
out.push_str(&quoted(rule));
}
Addressed::NotInHeaders => out.push_str("not-in-headers"),
}
out.push_str("\r\n");
}
out
}
fn hex(bytes: &[u8]) -> String {
bytes.iter().map(|b| format!("{b:02x}")).collect()
}
/// Whether a message can travel as 8bit: no NULs, no line past 998 bytes.
fn fits_8bit(message: &[u8]) -> bool {
!message.contains(&0) && message.split(|b| *b == b'\n').all(|l| l.len() <= 998)
/// What a body holds, which decides how it may travel.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Body {
SevenBit,
EightBit,
/// NULs, or a line past 998 bytes.
Binary,
}
/// The whole report: headers, the fields, then the original untouched.
/// `from` is the address the report is from; `host` names the server in its
/// Message-ID.
impl Body {
pub fn of(bytes: &[u8]) -> Body {
if bytes.contains(&0) || bytes.split(|b| *b == b'\n').any(|l| l.len() > 998) {
Body::Binary
} else if bytes.is_ascii() {
Body::SevenBit
} else {
Body::EightBit
}
}
fn encoding(&self) -> &'static str {
match self {
Body::SevenBit => "7bit",
Body::EightBit => "8bit",
Body::Binary => "binary",
}
}
}
/// The whole report: headers, the summary, the envelope, then the original
/// untouched. `from` is the address the report is from; `host` names the
/// server in its Message-ID and its `Reporting-MTA`.
pub fn build(
envelope: &Envelope<'_>,
original: &[u8],
@@ -196,7 +341,8 @@ pub fn build(
host: &str,
) -> (Vec<u8>, Fields) {
let fields = fields(envelope, original);
let body = text(envelope, &fields);
let summary = text(envelope, &fields);
let envelope_part = machine(envelope, &fields, host);
// A boundary that can't occur in the original
let mut boundary = format!("journal-{}", &hex(&Sha256::digest(original))[..32]);
while original
@@ -206,7 +352,8 @@ pub fn build(
boundary.push('x');
}
let mut out: Vec<u8> = Vec::with_capacity(original.len() + body.len() + 1024);
let mut out: Vec<u8> =
Vec::with_capacity(original.len() + summary.len() + envelope_part.len() + 1024);
out.extend_from_slice(format!("From: Journal <{}>\r\n", line(from)).as_bytes());
out.extend_from_slice(b"Date: ");
out.extend_from_slice(Date::new(envelope.received as i64).to_rfc822().as_bytes());
@@ -230,21 +377,42 @@ pub fn build(
out.extend_from_slice(format!("X-Inbuxa-Journal: {:x}\r\n", envelope.queue_id).as_bytes());
out.extend_from_slice(b"MIME-Version: 1.0\r\n");
out.extend_from_slice(
format!("Content-Type: multipart/mixed; boundary=\"{boundary}\"\r\n\r\n").as_bytes(),
format!(
"Content-Type: multipart/report; report-type=\"{REPORT_TYPE}\";\r\n\
\tboundary=\"{boundary}\"\r\n\r\n"
)
.as_bytes(),
);
out.extend_from_slice(format!("--{boundary}\r\n").as_bytes());
out.extend_from_slice(
b"Content-Type: text/plain; charset=utf-8\r\nContent-Transfer-Encoding: 8bit\r\n\r\n",
format!(
"Content-Type: text/plain; charset=utf-8\r\n\
Content-Transfer-Encoding: {}\r\n\r\n",
Body::of(summary.as_bytes()).encoding()
)
.as_bytes(),
);
out.extend_from_slice(body.as_bytes());
out.extend_from_slice(summary.as_bytes());
out.extend_from_slice(format!("\r\n--{boundary}\r\n").as_bytes());
out.extend_from_slice(
format!(
"Content-Type: {MEDIA_TYPE}\r\n\
Content-Transfer-Encoding: {}\r\n\r\n",
Body::of(envelope_part.as_bytes()).encoding()
)
.as_bytes(),
);
out.extend_from_slice(envelope_part.as_bytes());
out.extend_from_slice(format!("\r\n--{boundary}\r\n").as_bytes());
out.extend_from_slice(b"Content-Type: message/rfc822\r\n");
out.extend_from_slice(b"Content-Disposition: attachment; filename=\"original.eml\"\r\n");
out.extend_from_slice(if fits_8bit(original) {
b"Content-Transfer-Encoding: 8bit\r\n\r\n".as_slice()
} else {
b"Content-Transfer-Encoding: binary\r\n\r\n".as_slice()
});
out.extend_from_slice(
format!(
"Content-Transfer-Encoding: {}\r\n\r\n",
Body::of(original).encoding()
)
.as_bytes(),
);
out.extend_from_slice(original);
// The line break before a boundary belongs to the boundary: the
// original keeps its own last one
@@ -252,10 +420,17 @@ pub fn build(
(out, fields)
}
/// Where the original starts and ends inside a report [`build`] made.
/// Where the original starts and ends inside a report, in this layout or the
/// earlier one.
pub fn original(report: &[u8]) -> Option<&[u8]> {
let parsed = MessageParser::default().parse(report)?;
let part = parsed.attachment(0)?;
let part = parsed.attachments().find(|part| {
part.content_type().is_some_and(|c| {
c.ctype().eq_ignore_ascii_case("message")
&& c.subtype()
.is_some_and(|s| s.eq_ignore_ascii_case("rfc822"))
})
})?;
let start = part.raw_body_offset() as usize;
let end = part.raw_end_offset() as usize;
report.get(start..end)
@@ -264,6 +439,7 @@ pub fn original(report: &[u8]) -> Option<&[u8]> {
#[cfg(test)]
mod tests {
use super::*;
use mail_parser::PartType;
const ORIGINAL: &[u8] = b"From: [email protected]\r\n\
To: Bank <[email protected]>\r\n\
@@ -305,18 +481,16 @@ The figures.\r\n";
let fields = fields(&envelope(&recipients), ORIGINAL);
assert_eq!(fields.subject, "Q3 figures");
assert_eq!(fields.message_id, "<[email protected]>");
assert_eq!(fields.to, vec!["[email protected]"]);
assert_eq!(fields.cc, vec!["[email protected]"]);
assert_eq!(fields.bcc, vec!["[email protected]"]);
let list = Addressed::List("[email protected]".to_string());
assert_eq!(
fields.expanded,
vec![(
"finance@example.com".to_string(),
vec![
"dan@example.com".to_string(),
"eri[email protected]".to_string()
]
)]
fields.recipients,
vec![
("pay@bank.example".to_string(), Addressed::To),
("[email protected]".to_string(), Addressed::Cc),
("carol@example.com".to_string(), Addressed::NotInHeaders),
("da[email protected]".to_string(), list.clone()),
("[email protected]".to_string(), list),
]
);
}
@@ -334,7 +508,7 @@ The figures.\r\n";
);
let text = String::from_utf8_lossy(&report);
assert!(text.contains("Sender: [email protected]\r\n"));
assert!(text.contains("Bcc: [email protected]\r\n"));
assert!(text.contains("Not in To or Cc: [email protected]\r\n"));
assert!(text.contains("Queue ID: 1a2b\r\n"));
assert!(text.contains("Direction: outgoing\r\n"));
assert!(text.contains("Subject: Journal report: Q3 figures\r\n"));
@@ -350,6 +524,84 @@ The figures.\r\n";
assert_eq!(original(&report), Some(unterminated));
}
#[test]
fn report_has_the_three_parts_of_a_multipart_report() {
let recipients = [
rcpt("[email protected]", None),
rcpt("[email protected]", None),
];
let (report, _) = build(
&envelope(&recipients),
ORIGINAL,
"[email protected]",
"mx.example.com",
);
let parsed = MessageParser::default().parse(&report).unwrap();
let root = parsed.content_type().unwrap();
assert_eq!(root.ctype(), "multipart");
assert_eq!(root.subtype(), Some("report"));
assert_eq!(root.attribute("report-type"), Some(REPORT_TYPE));
let PartType::Multipart(children) = &parsed.parts[0].body else {
panic!("not a multipart");
};
let types: Vec<String> = children
.iter()
.map(|id| {
let c = parsed.parts[*id as usize].content_type().unwrap();
format!("{}/{}", c.ctype(), c.subtype().unwrap_or_default())
})
.collect();
assert_eq!(types, ["text/plain", MEDIA_TYPE, "message/rfc822"]);
}
#[test]
fn the_envelope_part_says_how_each_was_addressed() {
let mut copied = rcpt("[email protected]", None);
copied.added_by = Some("Copy \"finance\"".into());
let recipients = [
rcpt("[email protected]", None),
rcpt("[email protected]", None),
rcpt("[email protected]", None),
rcpt("[email protected]", Some("rfc822;[email protected]")),
copied,
];
let env = envelope(&recipients);
let fields = fields(&env, ORIGINAL);
let part = machine(&env, &fields, "mx.example.com");
let date = Date::new(env.received as i64).to_rfc822();
assert_eq!(
part,
format!(
"Version: 1\r\n\
Reporting-MTA: dns; mx.example.com\r\n\
Queue-ID: 1a2b\r\n\
Arrival-Date: {date}\r\n\
Direction: outgoing\r\n\
Authenticated: yes\r\n\
Held: no\r\n\
Original-Mail-From: <[email protected]>\r\n\
Original-Rcpt-To: <[email protected]>; addressed=to\r\n\
Original-Rcpt-To: <[email protected]>; addressed=cc\r\n\
Original-Rcpt-To: <[email protected]>; addressed=not-in-headers\r\n\
Original-Rcpt-To: <[email protected]>; addressed=list; via=<[email protected]>\r\n\
Original-Rcpt-To: <[email protected]>; addressed=rule; rule=\"Copy \\\"finance\\\"\"\r\n"
)
);
}
#[test]
fn an_orcpt_that_isnt_an_address_is_ignored() {
for orcpt in [
"rfc822;[email protected]>; addressed=to",
"rfc822;not an address",
"rfc822;nothing",
] {
let recipients = [rcpt("[email protected]", Some(orcpt))];
let fields = fields(&envelope(&recipients), ORIGINAL);
assert_eq!(fields.recipients[0].1, Addressed::NotInHeaders, "{orcpt}");
}
}
#[test]
fn rule_added_recipients_say_so() {
let mut copied = rcpt("[email protected]", None);
@@ -357,22 +609,26 @@ The figures.\r\n";
let recipients = [rcpt("[email protected]", None), copied];
let env = envelope(&recipients);
let fields = fields(&env, ORIGINAL);
assert!(fields.bcc.is_empty(), "{fields:?}");
assert!(
text(&env, &fields).contains("Added by rule: Copy finance -> [email protected]\r\n")
);
let body = text(&env, &fields);
assert!(!body.contains("Not in To or Cc"), "{body}");
assert!(body.contains("Added by rule: Copy finance -> [email protected]\r\n"));
}
#[test]
fn values_stay_on_one_line() {
let recipients = [rcpt("[email protected]", None)];
let mut env = envelope(&recipients);
env.sender = "[email protected]\r\nBcc: [email protected]";
env.sender = "[email protected]\r\nOriginal-Rcpt-To: <[email protected]>";
env.held = true;
let body = text(&env, &Fields::default());
assert_eq!(body.matches("\r\n").count(), body.lines().count());
assert!(body.contains("Sender: [email protected] Bcc: [email protected]\r\n"));
assert!(
body.contains("Sender: [email protected] Original-Rcpt-To: <[email protected]>\r\n")
);
assert!(body.contains("Held for review: yes\r\n"));
let part = machine(&env, &Fields::default(), "mx.example.com");
assert_eq!(part.matches("\r\nOriginal-Rcpt-To").count(), 0, "{part}");
assert!(part.contains("Held: yes\r\n"));
}
#[test]
@@ -381,5 +637,74 @@ The figures.\r\n";
let mut env = envelope(&recipients);
env.sender = "";
assert!(text(&env, &Fields::default()).starts_with("Sender: <>\r\n"));
assert!(
machine(&env, &Fields::default(), "mx.example.com")
.contains("Original-Mail-From: <>\r\n")
);
}
#[test]
fn a_long_list_or_subject_doesnt_make_the_report_binary() {
let recipients: Vec<Recipient> = (0..200)
.map(|n| rcpt(&format!("person{n}@example.com"), None))
.collect();
// Folded, so the original's own lines fit
let folded = vec!["é".repeat(100); 8].join("\r\n ");
let long = format!("Subject: {folded}\r\n\r\nBody\r\n");
let (report, fields) = build(
&envelope(&recipients),
long.as_bytes(),
"[email protected]",
"mx.example.com",
);
assert!(fields.subject.len() > SUMMARY_VALUE_MAX);
assert_eq!(Body::of(&report), Body::EightBit);
assert_eq!(original(&report), Some(long.as_bytes()));
}
#[test]
fn body_says_how_a_report_can_travel() {
assert_eq!(Body::of(b"plain\r\n"), Body::SevenBit);
assert_eq!(Body::of("caf\u{e9}\r\n".as_bytes()), Body::EightBit);
assert_eq!(Body::of(b"a\0b"), Body::Binary);
assert_eq!(Body::of("x".repeat(999).as_bytes()), Body::Binary);
assert_eq!(Body::of("x".repeat(998).as_bytes()), Body::SevenBit);
let recipients = [rcpt("[email protected]", None)];
let long = format!("Subject: x\r\n\r\n{}\r\n", "y".repeat(2000));
let (report, _) = build(
&envelope(&recipients),
long.as_bytes(),
"[email protected]",
"mx.example.com",
);
assert_eq!(Body::of(&report), Body::Binary);
let text = String::from_utf8_lossy(&report);
assert!(text.contains("Content-Transfer-Encoding: binary\r\n"));
assert_eq!(original(&report), Some(long.as_bytes()));
}
#[test]
fn reports_in_the_earlier_layout_still_give_their_original() {
let mut report = b"From: Journal <[email protected]>\r\n\
Subject: Journal report: Q3 figures\r\n\
MIME-Version: 1.0\r\n\
Content-Type: multipart/mixed; boundary=\"journal-old\"\r\n\
\r\n\
--journal-old\r\n\
Content-Type: text/plain; charset=utf-8\r\n\
Content-Transfer-Encoding: 8bit\r\n\
\r\n\
Sender: [email protected]\r\n\
Bcc: [email protected]\r\n\
\r\n\
--journal-old\r\n\
Content-Type: message/rfc822\r\n\
Content-Disposition: attachment; filename=\"original.eml\"\r\n\
Content-Transfer-Encoding: 8bit\r\n\
\r\n"
.to_vec();
report.extend_from_slice(ORIGINAL);
report.extend_from_slice(b"\r\n--journal-old--\r\n");
assert_eq!(original(&report), Some(ORIGINAL));
}
}
crates/smtp/src/outbound/session.rs
+121 -1
View File
@@ -10,13 +10,15 @@ use super::client::SmtpClient;
use crate::outbound::DeliveryResult;
use crate::outbound::client::{BoxResponse, from_error_status, from_mail_send_error};
use crate::outbound::error::ClientError;
use crate::queue::journal::FROM_JOURNAL;
use crate::queue::{Error, MessageWrapper, Recipient, Status, is_smtp_path};
use crate::queue::{ErrorDetails, HostResponse, UnexpectedResponse};
use common::Server;
use common::config::smtp::queue::ConnectionStrategy;
use directory::Credentials;
use smtp_proto::{
EXT_CHUNKING, EXT_DSN, EXT_REQUIRE_TLS, EXT_SIZE, EXT_SMTP_UTF8, EhloResponse, MAIL_REQUIRETLS,
EXT_8BIT_MIME, EXT_BINARY_MIME, EXT_CHUNKING, EXT_DSN, EXT_REQUIRE_TLS, EXT_SIZE,
EXT_SMTP_UTF8, EhloResponse, MAIL_BODY_8BITMIME, MAIL_BODY_BINARYMIME, MAIL_REQUIRETLS,
MAIL_RET_FULL, MAIL_RET_HDRS, MAIL_SMTPUTF8, RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE,
RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS, Severity,
};
@@ -121,6 +123,24 @@ impl MessageWrapper {
};*/
}
// inbuxa: a journal report's original can't be re-encoded, so a server
// that can't take its body whole doesn't get it. Refusing here keeps
// the report in the built-in journal (JR-7) rather than risk a changed
// copy in the archive.
if !self.journal_body_fits(&capabilities) {
smtp_client.quit().await;
statuses.push(DeliveryResult::domain(
Status::PermanentFailure(ErrorDetails {
entity: params.hostname.into(),
details: Error::Io(
"The server can't take this journal report's body unchanged".into(),
),
}),
rcpt_idxs,
));
return;
}
// MAIL FROM
let time = Instant::now();
smtp_client.timeout = params.conn_strategy.timeout_mail;
@@ -429,12 +449,37 @@ impl MessageWrapper {
smtp_client.quit().await;
}
/// inbuxa: whether the server can take a journal report's body as it is.
/// Every other message is sent as before.
fn journal_body_fits(&self, capabilities: &EhloResponse<String>) -> bool {
if !self.has_flag(FROM_JOURNAL) {
true
} else if self.has_flag(MAIL_BODY_BINARYMIME) {
// RFC 3030: BINARYMIME is only sent with BDAT
capabilities.has_capability(EXT_BINARY_MIME)
&& capabilities.has_capability(EXT_CHUNKING)
} else if self.has_flag(MAIL_BODY_8BITMIME) {
capabilities.has_capability(EXT_8BIT_MIME)
} else {
true
}
}
fn build_mail_from(&self, capabilities: &EhloResponse<String>) -> String {
let mut mail_from = String::with_capacity(self.message.return_path.len() + 60);
let _ = write!(mail_from, "MAIL FROM:<{}>", self.message.return_path);
if capabilities.has_capability(EXT_SIZE) {
let _ = write!(mail_from, " SIZE={}", self.message.size);
}
// inbuxa: journal reports declare their body (journal_body_fits has
// checked the server takes it)
if self.has_flag(FROM_JOURNAL) {
if self.has_flag(MAIL_BODY_BINARYMIME) {
mail_from.push_str(" BODY=BINARYMIME");
} else if self.has_flag(MAIL_BODY_8BITMIME) {
mail_from.push_str(" BODY=8BITMIME");
}
}
if self.has_flag(MAIL_REQUIRETLS) & capabilities.has_capability(EXT_REQUIRE_TLS) {
mail_from.push_str(" REQUIRETLS");
}
@@ -502,3 +547,78 @@ impl Recipient {
(self.flags & flag) != 0
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::queue::Message;
use common::config::smtp::queue::QueueName;
fn wrapper(flags: u64) -> MessageWrapper {
MessageWrapper {
queue_id: 0,
queue_name: QueueName::new("default").unwrap(),
is_multi_queue: false,
span_id: 0,
message: Message {
created: 0,
blob_hash: Default::default(),
return_path: "".into(),
recipients: vec![],
received_from_ip: "127.0.0.1".parse().unwrap(),
received_via_port: 0,
flags,
env_id: None,
priority: 0,
size: 0,
metadata: Box::new([]),
},
}
}
fn server(capabilities: u32) -> EhloResponse<String> {
let mut ehlo = EhloResponse::new("mx.example.org".to_string());
ehlo.capabilities = capabilities;
ehlo
}
#[test]
fn journal_reports_declare_their_body_where_it_fits() {
let eight = wrapper(FROM_JOURNAL | MAIL_BODY_8BITMIME);
let takes_8bit = server(EXT_8BIT_MIME);
assert!(eight.journal_body_fits(&takes_8bit));
assert!(
eight
.build_mail_from(&takes_8bit)
.contains(" BODY=8BITMIME")
);
assert!(!eight.journal_body_fits(&server(0)));
let binary = wrapper(FROM_JOURNAL | MAIL_BODY_BINARYMIME);
let takes_binary = server(EXT_8BIT_MIME | EXT_BINARY_MIME | EXT_CHUNKING);
assert!(binary.journal_body_fits(&takes_binary));
assert!(
binary
.build_mail_from(&takes_binary)
.contains(" BODY=BINARYMIME")
);
assert!(!binary.journal_body_fits(&takes_8bit));
// RFC 3030: no BDAT, no BINARYMIME
assert!(!binary.journal_body_fits(&server(EXT_8BIT_MIME | EXT_BINARY_MIME)));
let plain = wrapper(FROM_JOURNAL);
assert!(plain.journal_body_fits(&server(0)));
assert!(!plain.build_mail_from(&server(0)).contains("BODY="));
}
#[test]
fn other_mail_goes_out_as_it_did() {
let other = wrapper(MAIL_BODY_8BITMIME);
assert!(other.journal_body_fits(&server(0)));
assert!(
!other
.build_mail_from(&server(EXT_8BIT_MIME))
.contains("BODY=")
);
}
}
crates/smtp/src/queue/journal.rs
+7 -1
View File
@@ -25,6 +25,7 @@ use inbuxa_features::{
},
mailflow::held::HOLD_SECONDS,
};
use smtp_proto::{MAIL_BODY_8BITMIME, MAIL_BODY_BINARYMIME};
use store::write::{BatchBuilder, BlobLink, BlobOp, now};
use types::blob_hash::BlobHash;
@@ -204,7 +205,12 @@ pub async fn capture(
// From nobody: an archive's refusal comes back to no one, and the
// queue's own record of it is what counts (settle, below)
let mut report = server.new_message("", MessageSource::Autogenerated, 0);
report.message.flags |= FROM_JOURNAL;
report.message.flags |= FROM_JOURNAL
| match report::Body::of(&bytes) {
report::Body::SevenBit => 0,
report::Body::EightBit => MAIL_BODY_8BITMIME,
report::Body::Binary => MAIL_BODY_BINARYMIME,
};
report.add_expanded_recipient(&address, server).await;
let pending = Pending {
address,
docs/spec/features/journaling.md
+42 -20
View File
@@ -15,6 +15,7 @@ Written for the record SPEC.md §3 rule 3 asks for. Sources, and nothing else:
| `inbuxa-drafts/queue/journaling.md` | Own | What John asked for, and the gaps to settle |
| The DLP and mail flow rules spec, the audit-hold-lock spec, the personal-data catalog spec | Own | Conditions, the audit log, legal holds, roles, the catalog check |
| RFC 5321, RFC 3461 (DSN, ORCPT), RFC 2046 (`message/rfc822`), RFC 5322 | Public | The envelope, the original recipient of an expanded list, the report's shape |
| RFC 6522 (`multipart/report`), RFC 5965 (field names), RFC 8460 (the report-type registry), RFC 6152 and RFC 3030 (8BITMIME, BINARYMIME) | Public | The report's layout since the public specification |
No Enterprise-only file or snippet was used, and no third-party journaling
product or report format was consulted: the journal report below is our own
@@ -77,30 +78,37 @@ when it's queued, which is when it's held, with the hold noted in the entry.
### 2.2 The journal report (JR-3, JR-4)
**JR-3.** Each copy is a **journal report**: a new message whose first part
is `text/plain`, one field a line:
**JR-3.** Each copy is a **journal report**, as the public specification at
<https://coffeylabs.org/specs/journal-report/> defines it: a
`multipart/report` with `report-type="vnd.coffeylabs.journal-report"` and
three parts. The first is
a `text/plain` summary for people, not to be parsed. The second, of type
`application/vnd.coffeylabs.journal-report`, is the envelope for programs:
```
Sender: [email protected]
Signed in as: alice@example.com
Subject: Q3 figures
Message-ID: <…>
Queue ID: 1a2b3c…
Received: 2026-09-28T14:03:11Z
Version: 1
Reporting-MTA: dns; mx.example.com
Queue-ID: 1a2b3c
Arrival-Date: Mon, 28 Sep 2026 14:03:11 +0000
Direction: outgoing
To: [email protected]
Cc: [email protected]
Bcc: carol@example.com
Expanded: [email protected] -> dan@example.com, [email protected]
Held for review: yes
Authenticated: yes
Held: no
Original-Mail-From: <alice@example.com>
Original-Rcpt-To: <bank@elsewhere.example>; addressed=to
Original-Rcpt-To: <[email protected]>; addressed=not-in-headers
Original-Rcpt-To: <[email protected]>; addressed=list; via=<[email protected]>
Original-Rcpt-To: <[email protected]>; addressed=rule; rule="Copy finance"
```
and whose second part is the message as queued, **byte for byte**, as
`message/rfc822`. `Bcc:` lists envelope recipients that aren't in the
message's To or Cc headers. `Expanded:` groups the members of a list under
the list address, from their ORCPT. Recipients a transport rule added say so
(`Added by rule: <name>`). The field names are fixed English (they're a
record, not interface text), so a script can read them.
The third is the message as queued, **byte for byte**, as `message/rfc822`.
`addressed` says what the message and the envelope show: `not-in-headers`
means in neither To nor Cc, which a blind copy is, and so is a list's post
arriving for a subscriber, so the report doesn't call it Bcc. Field names are
fixed English (they're a record, not interface text).
Reports written before this layout are a `multipart/mixed` of a summary and
the message, and stay as they are: they're hashed into the chain. See
[As built](#as-built).
**JR-4.** One report per queued message, with the whole envelope, whatever
the scope matched on (§2.4). A message to 40 recipients is one report, not
@@ -278,7 +286,7 @@ fills in what it left open:
- **If the copy can't be taken**, the message isn't queued: the sender gets
a temporary failure and tries again. Nothing leaves unjournaled.
- **The report** says `Authenticated: yes|no` instead of the signed-in
account (the queue doesn't keep which account it was). `Added by rule`
account (the queue doesn't keep which account it was). `addressed=rule`
comes with **Journal it** in phase 3. A recipient given with an ORCPT
that names another address counts as expanded from that address.
- **Journal reports** the server queues carry message flag bit 48
@@ -347,6 +355,20 @@ Phase 5 (inbuxa-admin #62, server #119 for the menu, docs inbuxa.org #32):
- The rule editor's **Journal it**, on mail flow rules and as an optional
second action on DLP rules.
Public specification (`feat/journal-report-format`):
- **The layout** moved from a `multipart/mixed` to the `multipart/report`
JR-3 describes. The summary is now for people only, with one recipient a
line, and says "Not in To or Cc" where it said "Bcc". Reading the original
out of a report works for both layouts.
- **An ORCPT that isn't plainly an address is dropped.** A sender writes that
parameter, and it goes into a field programs read.
- **To an outside archive,** a report declares its body: `BODY=8BITMIME`, or
`BODY=BINARYMIME` for an original with a NUL or a line past 998 bytes. A
server that doesn't offer what's needed doesn't get the report, which counts
as refused and is kept under JR-7. Before, it was sent undeclared. Only
journal reports do this; other mail goes out as it did.
## Known gaps
- A message a person saves to Sent over IMAP, or sends through another
tests/src/system/journal.rs
+21 -2
View File
@@ -265,7 +265,20 @@ pub async fn test(test: &mut TestServer) {
"{text}"
);
assert!(
text.contains("Bcc: [email protected]\r\n"),
text.contains("Not in To or Cc: [email protected]\r\n"),
"{text}"
);
assert!(
text.contains(
"Content-Type: multipart/report; report-type=\"vnd.coffeylabs.journal-report\""
),
"{text}"
);
assert!(text.contains("Version: 1\r\n"), "{text}");
assert!(
text.contains(
"Original-Rcpt-To: <[email protected]>; addressed=not-in-headers\r\n"
),
"{text}"
);
let original = report::original(&bytes).expect("original part");
@@ -567,7 +580,13 @@ pub async fn archive(test: &mut TestServer) {
text.contains("Added by rule: Copy and journal -> [email protected]\r\n"),
"{text}"
);
assert!(!text.contains("Bcc:"), "{text}");
assert!(
text.contains(
"Original-Rcpt-To: <[email protected]>; addressed=rule; rule=\"Copy and journal\"\r\n"
),
"{text}"
);
assert!(!text.contains("Not in To or Cc"), "{text}");
// The same message went to the outside archive, which can't be reached
// from here: once it leaves the queue (given up on, or deleted), it's