Delivery history: show each message's subject #452

Merged
jcoffey-dev merged 1 commits from feat/228-trace-subject into main 2026-11-18 19:44:41 +00:00
14 changed files with 156 additions and 16 deletions

No files matched your search

+3 -1
View File
@@ -516,7 +516,9 @@ impl Tracers {
&& (Level::Info.is_contained(event_level)
|| event_type.is_span_start()
|| event_type.is_span_end()
|| event_type.as_str().starts_with("smtp.mail-from"))
|| event_type.as_str().starts_with("smtp.mail-from")
// inbuxa-admin #228: debug, so it reaches no info log
|| *event_type == EventType::Queue(trc::QueueEvent::MessageSubject))
{
interests.set(event_type.to_id() as usize);
global_interests.set(event_type.to_id() as usize);
+6 -4
View File
@@ -28,8 +28,8 @@ use store::{
write::{BatchBuilder, SearchIndex, TelemetryClass, ValueClass, now},
};
use trc::{
AddContext, AuthEvent, DeliveryEvent, Event, EventDetails, EventType, Key, Level, SmtpEvent,
ipc::subscriber::SubscriberBuilder,
AddContext, AuthEvent, DeliveryEvent, Event, EventDetails, EventType, Key, Level, QueueEvent,
SmtpEvent, ipc::subscriber::SubscriberBuilder,
};
use utils::snowflake::SnowflakeIdGenerator;
@@ -270,9 +270,11 @@ pub(crate) fn spawn_store_tracer(builder: SubscriberBuilder, tracing: Store, dat
span.has_mail_from = true;
}
let is_end = typ.is_span_end();
// MON-12: info and above, never raw I/O
// MON-12: info and above, never raw I/O; MON-42: the subject
if !typ.is_raw_io()
&& (is_end || event.inner.level as usize >= Level::Info as usize)
&& (is_end
|| event.inner.level as usize >= Level::Info as usize
|| typ == EventType::Queue(QueueEvent::MessageSubject))
{
if span.events.len() < MAX_EVENTS - 1 || is_end {
span.events.push(event);
+15 -1
View File
@@ -234,11 +234,13 @@ use store::{
use trc::{AddContext, EventType, Key};
/// The trace's server-set fields (MON-14): the first event's time, the
/// first `from`, every distinct `to`, and the message size or 0.
/// first `from`, every distinct `to`, the message size or 0, and the first
/// `subject` (MON-42).
fn trace_to_value(id: u64, trace: Trace) -> JmapValue<'static> {
let mut from = None;
let mut to: Vec<String> = Vec::new();
let mut size = None;
let mut subject = None;
let mut first_timestamp = None;
for event in trace.events.iter() {
first_timestamp.get_or_insert(event.timestamp.timestamp());
@@ -264,6 +266,11 @@ fn trace_to_value(id: u64, trace: Trace) -> JmapValue<'static> {
from = texts.into_iter().next();
}
}
Key::Subject => {
if subject.is_none() {
subject = texts.into_iter().next();
}
}
Key::To => {
for text in texts {
if !to.contains(&text) {
@@ -292,6 +299,13 @@ fn trace_to_value(id: u64, trace: Trace) -> JmapValue<'static> {
);
obj.insert_unchecked(Property::To, JmapValue::Str(to.join(", ").into()));
obj.insert_unchecked(Property::Size, JmapValue::Number(size.unwrap_or(0).into()));
obj.insert_unchecked(
Property::Subject,
match subject {
Some(subject) => JmapValue::Str(subject.into()),
None => JmapValue::Null,
},
);
}
value
}
+23
View File
@@ -57,6 +57,20 @@ impl QueuedMessage {
message.span_id = server.inner.data.span_id_gen.generate();
let span_id = message.span_id;
// inbuxa: the subject for delivery history (#228), read
// only when history is kept; a fresh message comes from
// this node's blob cache, and delivery reads it anyway
let subject = if !server.tracing_store().is_none() {
server
.get_message_blob(message.message.blob_hash.as_slice(), 0..usize::MAX)
.await
.ok()
.flatten()
.and_then(|blob| crate::queue::message_subject(&blob))
} else {
None
};
trc::event!(
Delivery(DeliveryEvent::AttemptStart),
SpanId = message.span_id,
@@ -86,6 +100,15 @@ impl QueuedMessage {
Size = message.message.size,
Total = message.message.recipients.len(),
);
// inbuxa: inbuxa-admin #228
if let Some(subject) = subject {
trc::event!(
Queue(trc::QueueEvent::MessageSubject),
SpanId = span_id,
QueueId = message.queue_id,
Subject = subject,
);
}
// Attempt delivery
let start_time = Instant::now();
+42
View File
@@ -87,6 +87,23 @@ pub struct Message {
pub metadata: Box<[Metadata]>,
}
/// inbuxa: the longest subject kept in delivery history (#228)
const SUBJECT_MAX_CHARS: usize = 255;
/// inbuxa: a message's subject for its delivery history (#228), from its
/// headers alone, cut to `SUBJECT_MAX_CHARS`. `None` when it has none.
pub fn message_subject(raw_message: &[u8]) -> Option<String> {
let parsed = mail_parser::MessageParser::new().parse_headers(raw_message)?;
let subject = parsed.subject()?.trim();
if subject.is_empty() {
return None;
}
Some(match subject.char_indices().nth(SUBJECT_MAX_CHARS) {
Some((end, _)) => format!("{}…", &subject[..end]),
None => subject.to_string(),
})
}
impl Message {
pub fn queued_event(&self) -> trc::QueueEvent {
if (self.flags & FROM_AUTHENTICATED) != 0 {
@@ -747,3 +764,28 @@ mod smtp_path_tests {
}
}
}
#[cfg(test)]
mod subject_tests {
use super::message_subject;
#[test]
fn subject_from_headers() {
assert_eq!(
message_subject(
b"From: [email protected]\r\nSubject: =?utf-8?q?Caf=C3=A9_menu?=\r\n\r\nbody"
)
.as_deref(),
Some("Café menu")
);
assert_eq!(
message_subject(b"From: [email protected]\r\n\r\nSubject: not a header"),
None
);
assert_eq!(message_subject(b"Subject: \r\n\r\n"), None);
let long = format!("Subject: {}\r\n\r\n", "é".repeat(300));
let subject = message_subject(long.as_bytes()).unwrap();
assert_eq!(subject.chars().count(), 256);
assert!(subject.ends_with('…'));
}
}
+16
View File
@@ -396,6 +396,13 @@ impl MessageWrapper {
..
} = params;
let event = self.message.queued_event();
// inbuxa: inbuxa-admin #228: the subject, only while delivery
// history is kept
let subject = if !server.tracing_store().is_none() {
crate::queue::message_subject(raw_message)
} else {
None
};
// Write blob
let raw_headers = raw_headers.unwrap_or_default();
@@ -503,6 +510,15 @@ impl MessageWrapper {
NextDsn = self.message.next_dsn(None).map(trc::Value::Timestamp),
Expires = self.message.expires(None).map(trc::Value::Timestamp),
);
// inbuxa: inbuxa-admin #228: the subject for delivery history
if let Some(subject) = subject {
trc::event!(
Queue(trc::QueueEvent::MessageSubject),
SpanId = session_id,
QueueId = self.queue_id,
Subject = subject,
);
}
// Write message to queue
let mut batch = BatchBuilder::new();
+7 -2
View File
@@ -15,10 +15,10 @@
// (audit-hold-lock spec, AU-3, AU-8); 649 is cluster.node-id-reassigned;
// 650 to 659 are inbuxa Relay's (relay spec, RL-23, RL-T23); 660 is
// store.blob-read-through; 661 is smtp.directory-no-answer (#364); 662 is
// spam.result (#397).
// spam.result (#397); 663 is queue.message-subject (inbuxa-admin #228).
// TOTAL_EVENT_COUNT sizes the collector's per-event tables, so it is one
// more than the highest id; the test below holds it to the variant list
pub const TOTAL_EVENT_COUNT: usize = 663;
pub const TOTAL_EVENT_COUNT: usize = 664;
pub const TOTAL_METRIC_COUNT: usize = 369;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -608,6 +608,9 @@ pub enum PushSubscriptionEvent {
pub enum QueueEvent {
Started = 390,
MessageQueued = 380,
/// inbuxa: inbuxa-admin #228: a queued message's subject, for delivery
/// history. Debug level, so an info log never holds subjects.
MessageSubject = 663,
AuthenticatedMessageQueued = 381,
ReportQueued = 382,
DsnQueued = 379,
@@ -1418,6 +1421,8 @@ pub enum Key {
Value = 63,
Version = 64,
QueueName = 65,
// inbuxa: a message's subject in delivery history (#228)
Subject = 66,
}
#[cfg(test)]
+18 -1
View File
@@ -426,6 +426,8 @@ impl EventType {
b"push-subscription.not-found" => EventType::PushSubscription(PushSubscriptionEvent::NotFound),
b"queue.started" => EventType::Queue(QueueEvent::Started),
b"queue.message-queued" => EventType::Queue(QueueEvent::MessageQueued),
// inbuxa: inbuxa-admin #228
b"queue.message-subject" => EventType::Queue(QueueEvent::MessageSubject),
b"queue.authenticated-message-queued" => EventType::Queue(QueueEvent::AuthenticatedMessageQueued),
b"queue.report-queued" => EventType::Queue(QueueEvent::ReportQueued),
b"queue.dsn-queued" => EventType::Queue(QueueEvent::DsnQueued),
@@ -1211,6 +1213,8 @@ impl EventType {
}
EventType::Queue(QueueEvent::Started) => "queue.started",
EventType::Queue(QueueEvent::MessageQueued) => "queue.message-queued",
// inbuxa: inbuxa-admin #228
EventType::Queue(QueueEvent::MessageSubject) => "queue.message-subject",
EventType::Queue(QueueEvent::AuthenticatedMessageQueued) => {
"queue.authenticated-message-queued"
}
@@ -1921,6 +1925,8 @@ impl EventType {
EventType::PushSubscription(PushSubscriptionEvent::NotFound) => 372,
EventType::Queue(QueueEvent::Started) => 390,
EventType::Queue(QueueEvent::MessageQueued) => 380,
// inbuxa: inbuxa-admin #228
EventType::Queue(QueueEvent::MessageSubject) => 663,
EventType::Queue(QueueEvent::AuthenticatedMessageQueued) => 381,
EventType::Queue(QueueEvent::ReportQueued) => 382,
EventType::Queue(QueueEvent::DsnQueued) => 379,
@@ -2635,6 +2641,8 @@ impl EventType {
372 => Some(EventType::PushSubscription(PushSubscriptionEvent::NotFound)),
390 => Some(EventType::Queue(QueueEvent::Started)),
380 => Some(EventType::Queue(QueueEvent::MessageQueued)),
// inbuxa: inbuxa-admin #228
663 => Some(EventType::Queue(QueueEvent::MessageSubject)),
381 => Some(EventType::Queue(QueueEvent::AuthenticatedMessageQueued)),
382 => Some(EventType::Queue(QueueEvent::ReportQueued)),
379 => Some(EventType::Queue(QueueEvent::DsnQueued)),
@@ -3081,6 +3089,8 @@ impl EventType {
EventType::OutgoingReport(OutgoingReportEvent::Locked) => Level::Info,
EventType::Queue(QueueEvent::Started) => Level::Info,
EventType::Queue(QueueEvent::MessageQueued) => Level::Info,
// inbuxa: inbuxa-admin #228: debug, so info logs never hold subjects
EventType::Queue(QueueEvent::MessageSubject) => Level::Debug,
EventType::Queue(QueueEvent::AuthenticatedMessageQueued) => Level::Info,
EventType::Queue(QueueEvent::ReportQueued) => Level::Info,
EventType::Queue(QueueEvent::DsnQueued) => Level::Info,
@@ -3816,6 +3826,8 @@ impl EventType {
}
EventType::Queue(QueueEvent::Started) => "MTA queue started",
EventType::Queue(QueueEvent::MessageQueued) => "Queued message for delivery",
// inbuxa: inbuxa-admin #228
EventType::Queue(QueueEvent::MessageSubject) => "Subject of the queued message",
EventType::Queue(QueueEvent::AuthenticatedMessageQueued) => {
"Queued message submission for delivery"
}
@@ -4830,6 +4842,8 @@ impl EventType {
EventType::PushSubscription(PushSubscriptionEvent::NotFound),
EventType::Queue(QueueEvent::Started),
EventType::Queue(QueueEvent::MessageQueued),
// inbuxa: inbuxa-admin #228
EventType::Queue(QueueEvent::MessageSubject),
EventType::Queue(QueueEvent::AuthenticatedMessageQueued),
EventType::Queue(QueueEvent::ReportQueued),
EventType::Queue(QueueEvent::DsnQueued),
@@ -8162,6 +8176,7 @@ impl Key {
b"value" => Key::Value,
b"version" => Key::Version,
b"queueName" => Key::QueueName,
b"subject" => Key::Subject,
}
.copied()
}
@@ -8234,6 +8249,7 @@ impl Key {
Key::Value => "value",
Key::Version => "version",
Key::QueueName => "queueName",
Key::Subject => "subject",
}
}
@@ -8309,11 +8325,12 @@ impl Key {
63 => Some(Key::Value),
64 => Some(Key::Version),
65 => Some(Key::QueueName),
66 => Some(Key::Subject),
_ => None,
}
}
pub const COUNT: usize = 66;
pub const COUNT: usize = 67;
}
impl serde::Serialize for Key {
+14 -2
View File
@@ -77,7 +77,7 @@ Clean-up of expired history runs on the existing `dataCleanupSchedule`.
`timestamp`, and `keyValues`: a list of `key` (a `Key`) and `value` (a
`TraceValue`: `String`, `UnsignedInt`, `Integer`, `Boolean`, `Float`,
`UTCDateTime`, `Duration`, `IpAddr`, `List`, `Event`, `Null`). Server-set:
`timestamp`, `from`, `to` (a string), `size`. Views: `x:Trace/InboundDelivery`
`timestamp`, `from`, `to` (a string), `subject` (MON-42), `size`. Views: `x:Trace/InboundDelivery`
and `x:Trace/OutboundDelivery`.
### The metric sample, `x:Metric` (read-only)
@@ -240,6 +240,18 @@ unchanged.
with its points, highest first (X-Spam-Result lists them lowest first). When one rule rejects or
discards a message by itself, `Details` names only that rule. An inbound
session's trace therefore shows why its message scored the way it did.
- **MON-42.** (inbuxa-admin #228) A trace's `subject` is the first `subject`
value in its events, or null. While a tracing store is set, the server
raises `queue.message-subject` (debug level, id 663) with the message's
`Subject` header, cut to 255 characters: in the SMTP session when it
queues a message, and in the delivery attempt right after
`delivery.attempt-start`, read from the message's blob (a fresh message
comes from the node's blob cache). The trace history keeps this one debug
event, so an info-level log or OpenTelemetry tracer never holds subjects;
a tracer set to debug does. With no tracing store nothing is read or
raised. Both list views show it between To and Size. It isn't in the
`text` search keywords. Like every other trace field it is kept for
`holdTracesFor`, and only the superuser set can read it.
### Retention and growth
@@ -567,7 +579,7 @@ they cover tests 10 and 12 (`tracing.rs`), 13 (`metrics.rs`) and 18
(`alerts.rs`). Tests 1, 3, 5 to 9, 11, 14 to 17, 19 to 21, 23 and 24 pass as
`tests/src/system/monitoring.rs`.
- **MON-1 to MON-41:** built, with the limits below.
- **MON-1 to MON-42:** built, with the limits below.
- **Test 26 (compat)** is written as `monitoring_compat`, ignored, and unrun
until a copy of INBUXA's data is provided. It checks observed 1, reads the
old history without an error, and purges it. It deletes history, so only
+1 -1
View File
@@ -463,7 +463,7 @@ Response `list[0]`:
{
"id": "x:Trace",
"kind": "object",
"categories": ["network", "identifier", "metadata"],
"categories": ["network", "identifier", "metadata", "content"],
"collected": true,
"retention": { "days": 30, "setting": "x:DataRetention.holdTracesFor" },
"leavesHost": false,
+5 -3
View File
@@ -2268,9 +2268,11 @@ where = ["data-store", "search-store"]
scope = "server"
retention = { setting = "x:DataRetention.holdTracesFor" }
[object."x:Trace".properties]
events = ["network", "identifier", "metadata"]
events = ["network", "identifier", "metadata", "content"]
from = ["identifier"]
size = ["metadata"]
# inbuxa-admin #228: the message's Subject header
subject = ["content"]
timestamp = ["metadata"]
to = ["identifier"]
@@ -2281,7 +2283,7 @@ where = ["data-store"]
scope = "server"
retention = { setting = "x:DataRetention.holdTracesFor" }
[object."x:TraceEvent".properties]
keyValues = ["network", "identifier", "metadata"]
keyValues = ["network", "identifier", "metadata", "content"]
timestamp = ["metadata"]
[object."x:TraceKeyValue"]
@@ -2291,7 +2293,7 @@ where = ["data-store"]
scope = "server"
retention = { setting = "x:DataRetention.holdTracesFor" }
[object."x:TraceKeyValue".properties]
value = ["network", "identifier", "metadata"]
value = ["network", "identifier", "metadata", "content"]
[object."x:TraceValueBoolean"]
default = "none"
Binary file not shown.
+1 -1
View File
@@ -1 +1 @@
UDW2s5eJMuFvg1FJkP9HP4a1IApTtrR0Vj9awc3VjhY
WWnyz0UpWEvf1aqR3HadgQyWuWDvGtqQXwpOgmnafSg
+5
View File
@@ -167,6 +167,11 @@ pub async fn test(test: &mut TestServer) {
.is_some_and(|to| to.contains("[email protected]"))),
"test 9: to"
);
// #228: the session and the delivery attempt both name the subject
assert!(
traces.iter().all(|t| t["subject"] == "Traced"),
"test 9: subject, {traces:?}"
);
// #397: the session came from outside and the attempt only filed the
// message here, so each list's default leaves out the other kind