From edf5a6441fc21db5dc03f9bbcfaee35adf3409b0 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Tue, 3 Nov 2026 21:13:25 -0700 Subject: [PATCH] Journal reports: move to a documented multipart/report Reports now follow coffeylabs.org/specs/journal-report/: a summary for people, an envelope part for programs with one line per recipient, and the original byte for byte. "Bcc" becomes not-in-headers, since a list's post looks the same. Stored reports keep their old layout, and the original can still be read from either. To an outside archive, a report now declares BODY=8BITMIME or BINARYMIME. A server that can't take the body whole doesn't get the report; it's kept in the built-in journal under JR-7. Only journal reports change here. An ORCPT that isn't plainly an address is dropped, because it goes into a field programs read. --- crates/features/src/journal/report.rs | 507 +++++++++++++++++++++----- crates/smtp/src/outbound/session.rs | 122 ++++++- crates/smtp/src/queue/journal.rs | 8 +- docs/spec/features/journaling.md | 62 +++- tests/src/system/journal.rs | 23 +- 5 files changed, 607 insertions(+), 115 deletions(-) diff --git a/crates/features/src/journal/report.rs b/crates/features/src/journal/report.rs index 8e385971..f28db40e 100644 --- a/crates/features/src/journal/report.rs +++ b/crates/features/src/journal/report.rs @@ -4,16 +4,35 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -//! The journal report (JR-3, JR-4): a message whose first part lists the -//! envelope, one field a line, and whose second part is the message as it -//! was queued, byte for byte, as `message/rfc822`. Field names are fixed -//! English: a report is a record, and scripts read it. +//! The journal report (JR-3, JR-4), as coffeylabs.org/specs/journal-report/ +//! defines it: a `multipart/report` whose first part is a summary for people, +//! whose second lists the envelope for programs, and whose third is the +//! message as it was queued, byte for byte, as `message/rfc822`. +//! +//! Reports already stored keep the layout they were written in, a +//! `multipart/mixed` of the summary and the message, and [`original`] reads +//! both. use super::Direction; use mail_builder::headers::{Header, date::Date, text::Text}; -use mail_parser::MessageParser; +use mail_parser::{MessageParser, MimeHeaders}; use sha2::{Digest, Sha256}; +/// The `report-type` of the `multipart/report`: RFC 6522 has it be the +/// subtype of the second part. +pub const REPORT_TYPE: &str = "vnd.coffeylabs.journal-report"; +/// The media type of the part programs read. +pub const MEDIA_TYPE: &str = "application/vnd.coffeylabs.journal-report"; +/// The version that part declares. +pub const VERSION: u32 = 1; + +/// The longest value the summary shows: a line of a message that travels as +/// 8bit can't pass 998 bytes, and the original holds the whole of it. +const SUMMARY_VALUE_MAX: usize = 900; +/// The longest address taken from a sender's ORCPT parameter (RFC 5321 +/// allows a path of 256; this leaves room for internationalized ones). +const VIA_MAX: usize = 320; + /// One envelope recipient, with the address it was given as (a list's, for /// the list's members). #[derive(Debug, Clone, PartialEq, Eq)] @@ -37,19 +56,27 @@ pub struct Envelope<'x> { pub held: bool, } +/// How a recipient came to be on the envelope, as far as the message and +/// the envelope show. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Addressed { + To, + Cc, + /// Reached through another address, such as a list's: the ORCPT named it. + List(String), + /// A mail flow rule added it or redirected to it. + Rule(String), + /// In neither To nor Cc, with nothing to say why. A blind copy looks like + /// this, and so does a list's post, or a message arriving for an alias. + NotInHeaders, +} + /// What a report says, besides the envelope's own fields. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct Fields { pub subject: String, pub message_id: String, - pub to: Vec, - pub cc: Vec, - /// Envelope recipients in neither To nor Cc, nor reached through a list. - pub bcc: Vec, - /// A list's address, and its members among the recipients. - pub expanded: Vec<(String, Vec)>, - /// A rule's name, and the recipients it added. - pub added: Vec<(String, Vec)>, + pub recipients: Vec<(String, Addressed)>, } /// One line's worth of a value: no line breaks, no control characters. @@ -62,14 +89,22 @@ fn line(value: &str) -> String { .to_string() } -/// The address an ORCPT names, without its `rfc822;` type. -fn orcpt_address(orcpt: &str) -> String { +/// The address an ORCPT names, without its `rfc822;` type. A sender writes +/// this parameter, so anything that isn't plainly an address is dropped: it +/// goes into a field programs read. +fn orcpt_address(orcpt: &str) -> Option { let orcpt = orcpt.trim(); let bare = match orcpt.split_once(';') { Some((kind, address)) if kind.eq_ignore_ascii_case("rfc822") => address, _ => orcpt, }; - bare.trim().to_lowercase() + let address = bare.trim().to_lowercase(); + let plain = address.len() <= VIA_MAX + && address.contains('@') + && !address.chars().any(|c| { + c.is_whitespace() || c.is_control() || matches!(c, '<' | '>' | ';' | '"' | ',') + }); + plain.then_some(address) } /// Sorts the envelope's recipients by how they were addressed. @@ -98,49 +133,59 @@ pub fn fields(envelope: &Envelope<'_>, original: &[u8]) -> Fields { None => Default::default(), }; - let mut fields = Fields { + let recipients = envelope + .recipients + .iter() + .map(|rcpt| { + let address = rcpt.address.to_lowercase(); + let via = rcpt + .orcpt + .as_deref() + .and_then(orcpt_address) + .filter(|via| *via != address); + let addressed = if let Some(rule) = &rcpt.added_by { + Addressed::Rule(line(rule)) + } else if header_to.contains(&address) { + Addressed::To + } else if header_cc.contains(&address) { + Addressed::Cc + } else if let Some(via) = via { + Addressed::List(via) + } else { + Addressed::NotInHeaders + }; + (line(&rcpt.address), addressed) + }) + .collect(); + + Fields { subject, message_id, - ..Default::default() - }; - for rcpt in envelope.recipients { - let address = rcpt.address.to_lowercase(); - let via = rcpt - .orcpt - .as_deref() - .map(orcpt_address) - .filter(|via| !via.is_empty() && *via != address); - if let Some(rule) = &rcpt.added_by { - match fields.added.iter_mut().find(|(name, _)| name == rule) { - Some((_, added)) => added.push(line(&rcpt.address)), - None => fields.added.push((line(rule), vec![line(&rcpt.address)])), - } - } else if header_to.contains(&address) { - fields.to.push(line(&rcpt.address)); - } else if header_cc.contains(&address) { - fields.cc.push(line(&rcpt.address)); - } else if let Some(via) = via { - match fields.expanded.iter_mut().find(|(list, _)| *list == via) { - Some((_, members)) => members.push(line(&rcpt.address)), - None => fields - .expanded - .push((line(&via), vec![line(&rcpt.address)])), - } - } else { - fields.bcc.push(line(&rcpt.address)); - } + recipients, } - fields } -/// The report's first part. +/// At most `max` bytes of a value, cut between characters. +fn fit(value: &str, max: usize) -> &str { + if value.len() <= max { + return value; + } + let mut end = max; + while !value.is_char_boundary(end) { + end -= 1; + } + &value[..end] +} + +/// The report's first part, for people. One recipient a line, so a long +/// list can't make a line too long to travel. Programs read the second part. pub fn text(envelope: &Envelope<'_>, fields: &Fields) -> String { let mut out = String::new(); let mut field = |name: &str, value: &str| { if !value.is_empty() { out.push_str(name); out.push_str(": "); - out.push_str(value); + out.push_str(fit(value, SUMMARY_VALUE_MAX)); out.push_str("\r\n"); } }; @@ -162,14 +207,32 @@ pub fn text(envelope: &Envelope<'_>, fields: &Fields) -> String { &mail_parser::DateTime::from_timestamp(envelope.received as i64).to_rfc3339(), ); field("Direction", envelope.direction.as_str()); - field("To", &fields.to.join(", ")); - field("Cc", &fields.cc.join(", ")); - field("Bcc", &fields.bcc.join(", ")); - for (list, members) in &fields.expanded { - field("Expanded", &format!("{list} -> {}", members.join(", "))); + let of = |wanted: &dyn Fn(&Addressed) -> bool| -> Vec<&str> { + fields + .recipients + .iter() + .filter(|(_, addressed)| wanted(addressed)) + .map(|(address, _)| address.as_str()) + .collect() + }; + for address in of(&|a| *a == Addressed::To) { + field("To", address); } - for (rule, added) in &fields.added { - field("Added by rule", &format!("{rule} -> {}", added.join(", "))); + for address in of(&|a| *a == Addressed::Cc) { + field("Cc", address); + } + for address in of(&|a| *a == Addressed::NotInHeaders) { + field("Not in To or Cc", address); + } + for (address, addressed) in &fields.recipients { + if let Addressed::List(via) = addressed { + field("Expanded", &format!("{via} -> {address}")); + } + } + for (address, addressed) in &fields.recipients { + if let Addressed::Rule(rule) = addressed { + field("Added by rule", &format!("{rule} -> {address}")); + } } if envelope.held { field("Held for review", "yes"); @@ -177,18 +240,100 @@ pub fn text(envelope: &Envelope<'_>, fields: &Fields) -> String { out } +/// A value of the second part's `rule` parameter. +fn quoted(value: &str) -> String { + let mut out = String::with_capacity(value.len() + 2); + out.push('"'); + for c in value.chars() { + if matches!(c, '"' | '\\') { + out.push('\\'); + } + out.push(c); + } + out.push('"'); + out +} + +/// The report's second part, for programs: one field a line, and one +/// `Original-Rcpt-To` for each envelope recipient. Field names are fixed +/// English, modeled on RFC 5965's where they mean the same. +pub fn machine(envelope: &Envelope<'_>, fields: &Fields, host: &str) -> String { + let mut out = format!( + "Version: {VERSION}\r\n\ + Reporting-MTA: dns; {}\r\n\ + Queue-ID: {:x}\r\n\ + Arrival-Date: {}\r\n\ + Direction: {}\r\n\ + Authenticated: {}\r\n\ + Held: {}\r\n\ + Original-Mail-From: <{}>\r\n", + line(host), + envelope.queue_id, + Date::new(envelope.received as i64).to_rfc822(), + envelope.direction.as_str(), + if envelope.authenticated { "yes" } else { "no" }, + if envelope.held { "yes" } else { "no" }, + line(envelope.sender), + ); + for (address, addressed) in &fields.recipients { + out.push_str("Original-Rcpt-To: <"); + out.push_str(address); + out.push_str(">; addressed="); + match addressed { + Addressed::To => out.push_str("to"), + Addressed::Cc => out.push_str("cc"), + Addressed::List(via) => { + out.push_str("list; via=<"); + out.push_str(via); + out.push('>'); + } + Addressed::Rule(rule) => { + out.push_str("rule; rule="); + out.push_str("ed(rule)); + } + Addressed::NotInHeaders => out.push_str("not-in-headers"), + } + out.push_str("\r\n"); + } + out +} + fn hex(bytes: &[u8]) -> String { bytes.iter().map(|b| format!("{b:02x}")).collect() } -/// Whether a message can travel as 8bit: no NULs, no line past 998 bytes. -fn fits_8bit(message: &[u8]) -> bool { - !message.contains(&0) && message.split(|b| *b == b'\n').all(|l| l.len() <= 998) +/// What a body holds, which decides how it may travel. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Body { + SevenBit, + EightBit, + /// NULs, or a line past 998 bytes. + Binary, } -/// The whole report: headers, the fields, then the original untouched. -/// `from` is the address the report is from; `host` names the server in its -/// Message-ID. +impl Body { + pub fn of(bytes: &[u8]) -> Body { + if bytes.contains(&0) || bytes.split(|b| *b == b'\n').any(|l| l.len() > 998) { + Body::Binary + } else if bytes.is_ascii() { + Body::SevenBit + } else { + Body::EightBit + } + } + + fn encoding(&self) -> &'static str { + match self { + Body::SevenBit => "7bit", + Body::EightBit => "8bit", + Body::Binary => "binary", + } + } +} + +/// The whole report: headers, the summary, the envelope, then the original +/// untouched. `from` is the address the report is from; `host` names the +/// server in its Message-ID and its `Reporting-MTA`. pub fn build( envelope: &Envelope<'_>, original: &[u8], @@ -196,7 +341,8 @@ pub fn build( host: &str, ) -> (Vec, Fields) { let fields = fields(envelope, original); - let body = text(envelope, &fields); + let summary = text(envelope, &fields); + let envelope_part = machine(envelope, &fields, host); // A boundary that can't occur in the original let mut boundary = format!("journal-{}", &hex(&Sha256::digest(original))[..32]); while original @@ -206,7 +352,8 @@ pub fn build( boundary.push('x'); } - let mut out: Vec = Vec::with_capacity(original.len() + body.len() + 1024); + let mut out: Vec = + Vec::with_capacity(original.len() + summary.len() + envelope_part.len() + 1024); out.extend_from_slice(format!("From: Journal <{}>\r\n", line(from)).as_bytes()); out.extend_from_slice(b"Date: "); out.extend_from_slice(Date::new(envelope.received as i64).to_rfc822().as_bytes()); @@ -230,21 +377,42 @@ pub fn build( out.extend_from_slice(format!("X-Inbuxa-Journal: {:x}\r\n", envelope.queue_id).as_bytes()); out.extend_from_slice(b"MIME-Version: 1.0\r\n"); out.extend_from_slice( - format!("Content-Type: multipart/mixed; boundary=\"{boundary}\"\r\n\r\n").as_bytes(), + format!( + "Content-Type: multipart/report; report-type=\"{REPORT_TYPE}\";\r\n\ + \tboundary=\"{boundary}\"\r\n\r\n" + ) + .as_bytes(), ); out.extend_from_slice(format!("--{boundary}\r\n").as_bytes()); out.extend_from_slice( - b"Content-Type: text/plain; charset=utf-8\r\nContent-Transfer-Encoding: 8bit\r\n\r\n", + format!( + "Content-Type: text/plain; charset=utf-8\r\n\ + Content-Transfer-Encoding: {}\r\n\r\n", + Body::of(summary.as_bytes()).encoding() + ) + .as_bytes(), ); - out.extend_from_slice(body.as_bytes()); + out.extend_from_slice(summary.as_bytes()); + out.extend_from_slice(format!("\r\n--{boundary}\r\n").as_bytes()); + out.extend_from_slice( + format!( + "Content-Type: {MEDIA_TYPE}\r\n\ + Content-Transfer-Encoding: {}\r\n\r\n", + Body::of(envelope_part.as_bytes()).encoding() + ) + .as_bytes(), + ); + out.extend_from_slice(envelope_part.as_bytes()); out.extend_from_slice(format!("\r\n--{boundary}\r\n").as_bytes()); out.extend_from_slice(b"Content-Type: message/rfc822\r\n"); out.extend_from_slice(b"Content-Disposition: attachment; filename=\"original.eml\"\r\n"); - out.extend_from_slice(if fits_8bit(original) { - b"Content-Transfer-Encoding: 8bit\r\n\r\n".as_slice() - } else { - b"Content-Transfer-Encoding: binary\r\n\r\n".as_slice() - }); + out.extend_from_slice( + format!( + "Content-Transfer-Encoding: {}\r\n\r\n", + Body::of(original).encoding() + ) + .as_bytes(), + ); out.extend_from_slice(original); // The line break before a boundary belongs to the boundary: the // original keeps its own last one @@ -252,10 +420,17 @@ pub fn build( (out, fields) } -/// Where the original starts and ends inside a report [`build`] made. +/// Where the original starts and ends inside a report, in this layout or the +/// earlier one. pub fn original(report: &[u8]) -> Option<&[u8]> { let parsed = MessageParser::default().parse(report)?; - let part = parsed.attachment(0)?; + let part = parsed.attachments().find(|part| { + part.content_type().is_some_and(|c| { + c.ctype().eq_ignore_ascii_case("message") + && c.subtype() + .is_some_and(|s| s.eq_ignore_ascii_case("rfc822")) + }) + })?; let start = part.raw_body_offset() as usize; let end = part.raw_end_offset() as usize; report.get(start..end) @@ -264,6 +439,7 @@ pub fn original(report: &[u8]) -> Option<&[u8]> { #[cfg(test)] mod tests { use super::*; + use mail_parser::PartType; const ORIGINAL: &[u8] = b"From: alice@example.com\r\n\ To: Bank \r\n\ @@ -305,18 +481,16 @@ The figures.\r\n"; let fields = fields(&envelope(&recipients), ORIGINAL); assert_eq!(fields.subject, "Q3 figures"); assert_eq!(fields.message_id, ""); - assert_eq!(fields.to, vec!["pay@bank.example"]); - assert_eq!(fields.cc, vec!["Bob@example.com"]); - assert_eq!(fields.bcc, vec!["carol@example.com"]); + let list = Addressed::List("finance@example.com".to_string()); assert_eq!( - fields.expanded, - vec![( - "finance@example.com".to_string(), - vec![ - "dan@example.com".to_string(), - "erin@example.com".to_string() - ] - )] + fields.recipients, + vec![ + ("pay@bank.example".to_string(), Addressed::To), + ("Bob@example.com".to_string(), Addressed::Cc), + ("carol@example.com".to_string(), Addressed::NotInHeaders), + ("dan@example.com".to_string(), list.clone()), + ("erin@example.com".to_string(), list), + ] ); } @@ -334,7 +508,7 @@ The figures.\r\n"; ); let text = String::from_utf8_lossy(&report); assert!(text.contains("Sender: alice@example.com\r\n")); - assert!(text.contains("Bcc: carol@example.com\r\n")); + assert!(text.contains("Not in To or Cc: carol@example.com\r\n")); assert!(text.contains("Queue ID: 1a2b\r\n")); assert!(text.contains("Direction: outgoing\r\n")); assert!(text.contains("Subject: Journal report: Q3 figures\r\n")); @@ -350,6 +524,84 @@ The figures.\r\n"; assert_eq!(original(&report), Some(unterminated)); } + #[test] + fn report_has_the_three_parts_of_a_multipart_report() { + let recipients = [ + rcpt("pay@bank.example", None), + rcpt("carol@example.com", None), + ]; + let (report, _) = build( + &envelope(&recipients), + ORIGINAL, + "postmaster@example.com", + "mx.example.com", + ); + let parsed = MessageParser::default().parse(&report).unwrap(); + let root = parsed.content_type().unwrap(); + assert_eq!(root.ctype(), "multipart"); + assert_eq!(root.subtype(), Some("report")); + assert_eq!(root.attribute("report-type"), Some(REPORT_TYPE)); + let PartType::Multipart(children) = &parsed.parts[0].body else { + panic!("not a multipart"); + }; + let types: Vec = children + .iter() + .map(|id| { + let c = parsed.parts[*id as usize].content_type().unwrap(); + format!("{}/{}", c.ctype(), c.subtype().unwrap_or_default()) + }) + .collect(); + assert_eq!(types, ["text/plain", MEDIA_TYPE, "message/rfc822"]); + } + + #[test] + fn the_envelope_part_says_how_each_was_addressed() { + let mut copied = rcpt("archive@example.com", None); + copied.added_by = Some("Copy \"finance\"".into()); + let recipients = [ + rcpt("pay@bank.example", None), + rcpt("bob@example.com", None), + rcpt("carol@example.com", None), + rcpt("dan@example.com", Some("rfc822;finance@example.com")), + copied, + ]; + let env = envelope(&recipients); + let fields = fields(&env, ORIGINAL); + let part = machine(&env, &fields, "mx.example.com"); + let date = Date::new(env.received as i64).to_rfc822(); + assert_eq!( + part, + format!( + "Version: 1\r\n\ + Reporting-MTA: dns; mx.example.com\r\n\ + Queue-ID: 1a2b\r\n\ + Arrival-Date: {date}\r\n\ + Direction: outgoing\r\n\ + Authenticated: yes\r\n\ + Held: no\r\n\ + Original-Mail-From: \r\n\ + Original-Rcpt-To: ; addressed=to\r\n\ + Original-Rcpt-To: ; addressed=cc\r\n\ + Original-Rcpt-To: ; addressed=not-in-headers\r\n\ + Original-Rcpt-To: ; addressed=list; via=\r\n\ + Original-Rcpt-To: ; addressed=rule; rule=\"Copy \\\"finance\\\"\"\r\n" + ) + ); + } + + #[test] + fn an_orcpt_that_isnt_an_address_is_ignored() { + for orcpt in [ + "rfc822;x@example.com>; addressed=to", + "rfc822;not an address", + "rfc822;nothing", + ] { + let recipients = [rcpt("carol@example.com", Some(orcpt))]; + let fields = fields(&envelope(&recipients), ORIGINAL); + assert_eq!(fields.recipients[0].1, Addressed::NotInHeaders, "{orcpt}"); + } + } + #[test] fn rule_added_recipients_say_so() { let mut copied = rcpt("archive@example.com", None); @@ -357,22 +609,26 @@ The figures.\r\n"; let recipients = [rcpt("pay@bank.example", None), copied]; let env = envelope(&recipients); let fields = fields(&env, ORIGINAL); - assert!(fields.bcc.is_empty(), "{fields:?}"); - assert!( - text(&env, &fields).contains("Added by rule: Copy finance -> archive@example.com\r\n") - ); + let body = text(&env, &fields); + assert!(!body.contains("Not in To or Cc"), "{body}"); + assert!(body.contains("Added by rule: Copy finance -> archive@example.com\r\n")); } #[test] fn values_stay_on_one_line() { let recipients = [rcpt("x@example.com", None)]; let mut env = envelope(&recipients); - env.sender = "evil@example.com\r\nBcc: nobody@example.com"; + env.sender = "evil@example.com\r\nOriginal-Rcpt-To: "; env.held = true; let body = text(&env, &Fields::default()); assert_eq!(body.matches("\r\n").count(), body.lines().count()); - assert!(body.contains("Sender: evil@example.com Bcc: nobody@example.com\r\n")); + assert!( + body.contains("Sender: evil@example.com Original-Rcpt-To: \r\n") + ); assert!(body.contains("Held for review: yes\r\n")); + let part = machine(&env, &Fields::default(), "mx.example.com"); + assert_eq!(part.matches("\r\nOriginal-Rcpt-To").count(), 0, "{part}"); + assert!(part.contains("Held: yes\r\n")); } #[test] @@ -381,5 +637,74 @@ The figures.\r\n"; let mut env = envelope(&recipients); env.sender = ""; assert!(text(&env, &Fields::default()).starts_with("Sender: <>\r\n")); + assert!( + machine(&env, &Fields::default(), "mx.example.com") + .contains("Original-Mail-From: <>\r\n") + ); + } + + #[test] + fn a_long_list_or_subject_doesnt_make_the_report_binary() { + let recipients: Vec = (0..200) + .map(|n| rcpt(&format!("person{n}@example.com"), None)) + .collect(); + // Folded, so the original's own lines fit + let folded = vec!["é".repeat(100); 8].join("\r\n "); + let long = format!("Subject: {folded}\r\n\r\nBody\r\n"); + let (report, fields) = build( + &envelope(&recipients), + long.as_bytes(), + "postmaster@example.com", + "mx.example.com", + ); + assert!(fields.subject.len() > SUMMARY_VALUE_MAX); + assert_eq!(Body::of(&report), Body::EightBit); + assert_eq!(original(&report), Some(long.as_bytes())); + } + + #[test] + fn body_says_how_a_report_can_travel() { + assert_eq!(Body::of(b"plain\r\n"), Body::SevenBit); + assert_eq!(Body::of("caf\u{e9}\r\n".as_bytes()), Body::EightBit); + assert_eq!(Body::of(b"a\0b"), Body::Binary); + assert_eq!(Body::of("x".repeat(999).as_bytes()), Body::Binary); + assert_eq!(Body::of("x".repeat(998).as_bytes()), Body::SevenBit); + let recipients = [rcpt("x@example.com", None)]; + let long = format!("Subject: x\r\n\r\n{}\r\n", "y".repeat(2000)); + let (report, _) = build( + &envelope(&recipients), + long.as_bytes(), + "postmaster@example.com", + "mx.example.com", + ); + assert_eq!(Body::of(&report), Body::Binary); + let text = String::from_utf8_lossy(&report); + assert!(text.contains("Content-Transfer-Encoding: binary\r\n")); + assert_eq!(original(&report), Some(long.as_bytes())); + } + + #[test] + fn reports_in_the_earlier_layout_still_give_their_original() { + let mut report = b"From: Journal \r\n\ +Subject: Journal report: Q3 figures\r\n\ +MIME-Version: 1.0\r\n\ +Content-Type: multipart/mixed; boundary=\"journal-old\"\r\n\ +\r\n\ +--journal-old\r\n\ +Content-Type: text/plain; charset=utf-8\r\n\ +Content-Transfer-Encoding: 8bit\r\n\ +\r\n\ +Sender: alice@example.com\r\n\ +Bcc: carol@example.com\r\n\ +\r\n\ +--journal-old\r\n\ +Content-Type: message/rfc822\r\n\ +Content-Disposition: attachment; filename=\"original.eml\"\r\n\ +Content-Transfer-Encoding: 8bit\r\n\ +\r\n" + .to_vec(); + report.extend_from_slice(ORIGINAL); + report.extend_from_slice(b"\r\n--journal-old--\r\n"); + assert_eq!(original(&report), Some(ORIGINAL)); } } diff --git a/crates/smtp/src/outbound/session.rs b/crates/smtp/src/outbound/session.rs index 878df39c..0c24bdfc 100644 --- a/crates/smtp/src/outbound/session.rs +++ b/crates/smtp/src/outbound/session.rs @@ -10,13 +10,15 @@ use super::client::SmtpClient; use crate::outbound::DeliveryResult; use crate::outbound::client::{BoxResponse, from_error_status, from_mail_send_error}; use crate::outbound::error::ClientError; +use crate::queue::journal::FROM_JOURNAL; use crate::queue::{Error, MessageWrapper, Recipient, Status, is_smtp_path}; use crate::queue::{ErrorDetails, HostResponse, UnexpectedResponse}; use common::Server; use common::config::smtp::queue::ConnectionStrategy; use directory::Credentials; use smtp_proto::{ - EXT_CHUNKING, EXT_DSN, EXT_REQUIRE_TLS, EXT_SIZE, EXT_SMTP_UTF8, EhloResponse, MAIL_REQUIRETLS, + EXT_8BIT_MIME, EXT_BINARY_MIME, EXT_CHUNKING, EXT_DSN, EXT_REQUIRE_TLS, EXT_SIZE, + EXT_SMTP_UTF8, EhloResponse, MAIL_BODY_8BITMIME, MAIL_BODY_BINARYMIME, MAIL_REQUIRETLS, MAIL_RET_FULL, MAIL_RET_HDRS, MAIL_SMTPUTF8, RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS, Severity, }; @@ -121,6 +123,24 @@ impl MessageWrapper { };*/ } + // inbuxa: a journal report's original can't be re-encoded, so a server + // that can't take its body whole doesn't get it. Refusing here keeps + // the report in the built-in journal (JR-7) rather than risk a changed + // copy in the archive. + if !self.journal_body_fits(&capabilities) { + smtp_client.quit().await; + statuses.push(DeliveryResult::domain( + Status::PermanentFailure(ErrorDetails { + entity: params.hostname.into(), + details: Error::Io( + "The server can't take this journal report's body unchanged".into(), + ), + }), + rcpt_idxs, + )); + return; + } + // MAIL FROM let time = Instant::now(); smtp_client.timeout = params.conn_strategy.timeout_mail; @@ -429,12 +449,37 @@ impl MessageWrapper { smtp_client.quit().await; } + /// inbuxa: whether the server can take a journal report's body as it is. + /// Every other message is sent as before. + fn journal_body_fits(&self, capabilities: &EhloResponse) -> bool { + if !self.has_flag(FROM_JOURNAL) { + true + } else if self.has_flag(MAIL_BODY_BINARYMIME) { + // RFC 3030: BINARYMIME is only sent with BDAT + capabilities.has_capability(EXT_BINARY_MIME) + && capabilities.has_capability(EXT_CHUNKING) + } else if self.has_flag(MAIL_BODY_8BITMIME) { + capabilities.has_capability(EXT_8BIT_MIME) + } else { + true + } + } + fn build_mail_from(&self, capabilities: &EhloResponse) -> String { let mut mail_from = String::with_capacity(self.message.return_path.len() + 60); let _ = write!(mail_from, "MAIL FROM:<{}>", self.message.return_path); if capabilities.has_capability(EXT_SIZE) { let _ = write!(mail_from, " SIZE={}", self.message.size); } + // inbuxa: journal reports declare their body (journal_body_fits has + // checked the server takes it) + if self.has_flag(FROM_JOURNAL) { + if self.has_flag(MAIL_BODY_BINARYMIME) { + mail_from.push_str(" BODY=BINARYMIME"); + } else if self.has_flag(MAIL_BODY_8BITMIME) { + mail_from.push_str(" BODY=8BITMIME"); + } + } if self.has_flag(MAIL_REQUIRETLS) & capabilities.has_capability(EXT_REQUIRE_TLS) { mail_from.push_str(" REQUIRETLS"); } @@ -502,3 +547,78 @@ impl Recipient { (self.flags & flag) != 0 } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::queue::Message; + use common::config::smtp::queue::QueueName; + + fn wrapper(flags: u64) -> MessageWrapper { + MessageWrapper { + queue_id: 0, + queue_name: QueueName::new("default").unwrap(), + is_multi_queue: false, + span_id: 0, + message: Message { + created: 0, + blob_hash: Default::default(), + return_path: "".into(), + recipients: vec![], + received_from_ip: "127.0.0.1".parse().unwrap(), + received_via_port: 0, + flags, + env_id: None, + priority: 0, + size: 0, + metadata: Box::new([]), + }, + } + } + + fn server(capabilities: u32) -> EhloResponse { + let mut ehlo = EhloResponse::new("mx.example.org".to_string()); + ehlo.capabilities = capabilities; + ehlo + } + + #[test] + fn journal_reports_declare_their_body_where_it_fits() { + let eight = wrapper(FROM_JOURNAL | MAIL_BODY_8BITMIME); + let takes_8bit = server(EXT_8BIT_MIME); + assert!(eight.journal_body_fits(&takes_8bit)); + assert!( + eight + .build_mail_from(&takes_8bit) + .contains(" BODY=8BITMIME") + ); + assert!(!eight.journal_body_fits(&server(0))); + + let binary = wrapper(FROM_JOURNAL | MAIL_BODY_BINARYMIME); + let takes_binary = server(EXT_8BIT_MIME | EXT_BINARY_MIME | EXT_CHUNKING); + assert!(binary.journal_body_fits(&takes_binary)); + assert!( + binary + .build_mail_from(&takes_binary) + .contains(" BODY=BINARYMIME") + ); + assert!(!binary.journal_body_fits(&takes_8bit)); + // RFC 3030: no BDAT, no BINARYMIME + assert!(!binary.journal_body_fits(&server(EXT_8BIT_MIME | EXT_BINARY_MIME))); + + let plain = wrapper(FROM_JOURNAL); + assert!(plain.journal_body_fits(&server(0))); + assert!(!plain.build_mail_from(&server(0)).contains("BODY=")); + } + + #[test] + fn other_mail_goes_out_as_it_did() { + let other = wrapper(MAIL_BODY_8BITMIME); + assert!(other.journal_body_fits(&server(0))); + assert!( + !other + .build_mail_from(&server(EXT_8BIT_MIME)) + .contains("BODY=") + ); + } +} diff --git a/crates/smtp/src/queue/journal.rs b/crates/smtp/src/queue/journal.rs index a25c9cab..a8d78322 100644 --- a/crates/smtp/src/queue/journal.rs +++ b/crates/smtp/src/queue/journal.rs @@ -25,6 +25,7 @@ use inbuxa_features::{ }, mailflow::held::HOLD_SECONDS, }; +use smtp_proto::{MAIL_BODY_8BITMIME, MAIL_BODY_BINARYMIME}; use store::write::{BatchBuilder, BlobLink, BlobOp, now}; use types::blob_hash::BlobHash; @@ -204,7 +205,12 @@ pub async fn capture( // From nobody: an archive's refusal comes back to no one, and the // queue's own record of it is what counts (settle, below) let mut report = server.new_message("", MessageSource::Autogenerated, 0); - report.message.flags |= FROM_JOURNAL; + report.message.flags |= FROM_JOURNAL + | match report::Body::of(&bytes) { + report::Body::SevenBit => 0, + report::Body::EightBit => MAIL_BODY_8BITMIME, + report::Body::Binary => MAIL_BODY_BINARYMIME, + }; report.add_expanded_recipient(&address, server).await; let pending = Pending { address, diff --git a/docs/spec/features/journaling.md b/docs/spec/features/journaling.md index 0fc5e396..1700e56d 100644 --- a/docs/spec/features/journaling.md +++ b/docs/spec/features/journaling.md @@ -15,6 +15,7 @@ Written for the record SPEC.md §3 rule 3 asks for. Sources, and nothing else: | `inbuxa-drafts/queue/journaling.md` | Own | What John asked for, and the gaps to settle | | The DLP and mail flow rules spec, the audit-hold-lock spec, the personal-data catalog spec | Own | Conditions, the audit log, legal holds, roles, the catalog check | | RFC 5321, RFC 3461 (DSN, ORCPT), RFC 2046 (`message/rfc822`), RFC 5322 | Public | The envelope, the original recipient of an expanded list, the report's shape | +| RFC 6522 (`multipart/report`), RFC 5965 (field names), RFC 8460 (the report-type registry), RFC 6152 and RFC 3030 (8BITMIME, BINARYMIME) | Public | The report's layout since the public specification | No Enterprise-only file or snippet was used, and no third-party journaling product or report format was consulted: the journal report below is our own @@ -77,30 +78,37 @@ when it's queued, which is when it's held, with the hold noted in the entry. ### 2.2 The journal report (JR-3, JR-4) -**JR-3.** Each copy is a **journal report**: a new message whose first part -is `text/plain`, one field a line: +**JR-3.** Each copy is a **journal report**, as the public specification at + defines it: a +`multipart/report` with `report-type="vnd.coffeylabs.journal-report"` and +three parts. The first is +a `text/plain` summary for people, not to be parsed. The second, of type +`application/vnd.coffeylabs.journal-report`, is the envelope for programs: ``` -Sender: alice@example.com -Signed in as: alice@example.com -Subject: Q3 figures -Message-ID: <…> -Queue ID: 1a2b3c… -Received: 2026-09-28T14:03:11Z +Version: 1 +Reporting-MTA: dns; mx.example.com +Queue-ID: 1a2b3c +Arrival-Date: Mon, 28 Sep 2026 14:03:11 +0000 Direction: outgoing -To: bank@elsewhere.example -Cc: bob@example.com -Bcc: carol@example.com -Expanded: finance@example.com -> dan@example.com, erin@example.com -Held for review: yes +Authenticated: yes +Held: no +Original-Mail-From: +Original-Rcpt-To: ; addressed=to +Original-Rcpt-To: ; addressed=not-in-headers +Original-Rcpt-To: ; addressed=list; via= +Original-Rcpt-To: ; addressed=rule; rule="Copy finance" ``` -and whose second part is the message as queued, **byte for byte**, as -`message/rfc822`. `Bcc:` lists envelope recipients that aren't in the -message's To or Cc headers. `Expanded:` groups the members of a list under -the list address, from their ORCPT. Recipients a transport rule added say so -(`Added by rule: `). The field names are fixed English (they're a -record, not interface text), so a script can read them. +The third is the message as queued, **byte for byte**, as `message/rfc822`. +`addressed` says what the message and the envelope show: `not-in-headers` +means in neither To nor Cc, which a blind copy is, and so is a list's post +arriving for a subscriber, so the report doesn't call it Bcc. Field names are +fixed English (they're a record, not interface text). + +Reports written before this layout are a `multipart/mixed` of a summary and +the message, and stay as they are: they're hashed into the chain. See +[As built](#as-built). **JR-4.** One report per queued message, with the whole envelope, whatever the scope matched on (§2.4). A message to 40 recipients is one report, not @@ -278,7 +286,7 @@ fills in what it left open: - **If the copy can't be taken**, the message isn't queued: the sender gets a temporary failure and tries again. Nothing leaves unjournaled. - **The report** says `Authenticated: yes|no` instead of the signed-in - account (the queue doesn't keep which account it was). `Added by rule` + account (the queue doesn't keep which account it was). `addressed=rule` comes with **Journal it** in phase 3. A recipient given with an ORCPT that names another address counts as expanded from that address. - **Journal reports** the server queues carry message flag bit 48 @@ -347,6 +355,20 @@ Phase 5 (inbuxa-admin #62, server #119 for the menu, docs inbuxa.org #32): - The rule editor's **Journal it**, on mail flow rules and as an optional second action on DLP rules. +Public specification (`feat/journal-report-format`): + +- **The layout** moved from a `multipart/mixed` to the `multipart/report` + JR-3 describes. The summary is now for people only, with one recipient a + line, and says "Not in To or Cc" where it said "Bcc". Reading the original + out of a report works for both layouts. +- **An ORCPT that isn't plainly an address is dropped.** A sender writes that + parameter, and it goes into a field programs read. +- **To an outside archive,** a report declares its body: `BODY=8BITMIME`, or + `BODY=BINARYMIME` for an original with a NUL or a line past 998 bytes. A + server that doesn't offer what's needed doesn't get the report, which counts + as refused and is kept under JR-7. Before, it was sent undeclared. Only + journal reports do this; other mail goes out as it did. + ## Known gaps - A message a person saves to Sent over IMAP, or sends through another diff --git a/tests/src/system/journal.rs b/tests/src/system/journal.rs index 9367d085..e486f79e 100644 --- a/tests/src/system/journal.rs +++ b/tests/src/system/journal.rs @@ -265,7 +265,20 @@ pub async fn test(test: &mut TestServer) { "{text}" ); assert!( - text.contains("Bcc: journal-other@example.com\r\n"), + text.contains("Not in To or Cc: journal-other@example.com\r\n"), + "{text}" + ); + assert!( + text.contains( + "Content-Type: multipart/report; report-type=\"vnd.coffeylabs.journal-report\"" + ), + "{text}" + ); + assert!(text.contains("Version: 1\r\n"), "{text}"); + assert!( + text.contains( + "Original-Rcpt-To: ; addressed=not-in-headers\r\n" + ), "{text}" ); let original = report::original(&bytes).expect("original part"); @@ -567,7 +580,13 @@ pub async fn archive(test: &mut TestServer) { text.contains("Added by rule: Copy and journal -> journal-vault@example.com\r\n"), "{text}" ); - assert!(!text.contains("Bcc:"), "{text}"); + assert!( + text.contains( + "Original-Rcpt-To: ; addressed=rule; rule=\"Copy and journal\"\r\n" + ), + "{text}" + ); + assert!(!text.contains("Not in To or Cc"), "{text}"); // The same message went to the outside archive, which can't be reached // from here: once it leaves the queue (given up on, or deleted), it's -- 2.54.0