diff --git a/crates/common/src/config/telemetry.rs b/crates/common/src/config/telemetry.rs index 67e49722..28865310 100644 --- a/crates/common/src/config/telemetry.rs +++ b/crates/common/src/config/telemetry.rs @@ -516,7 +516,9 @@ impl Tracers { && (Level::Info.is_contained(event_level) || event_type.is_span_start() || event_type.is_span_end() - || event_type.as_str().starts_with("smtp.mail-from")) + || event_type.as_str().starts_with("smtp.mail-from") + // inbuxa-admin #228: debug, so it reaches no info log + || *event_type == EventType::Queue(trc::QueueEvent::MessageSubject)) { interests.set(event_type.to_id() as usize); global_interests.set(event_type.to_id() as usize); diff --git a/crates/common/src/telemetry/tracers/store.rs b/crates/common/src/telemetry/tracers/store.rs index 2103b8aa..adc5cec6 100644 --- a/crates/common/src/telemetry/tracers/store.rs +++ b/crates/common/src/telemetry/tracers/store.rs @@ -28,8 +28,8 @@ use store::{ write::{BatchBuilder, SearchIndex, TelemetryClass, ValueClass, now}, }; use trc::{ - AddContext, AuthEvent, DeliveryEvent, Event, EventDetails, EventType, Key, Level, SmtpEvent, - ipc::subscriber::SubscriberBuilder, + AddContext, AuthEvent, DeliveryEvent, Event, EventDetails, EventType, Key, Level, QueueEvent, + SmtpEvent, ipc::subscriber::SubscriberBuilder, }; use utils::snowflake::SnowflakeIdGenerator; @@ -270,9 +270,11 @@ pub(crate) fn spawn_store_tracer(builder: SubscriberBuilder, tracing: Store, dat span.has_mail_from = true; } let is_end = typ.is_span_end(); - // MON-12: info and above, never raw I/O + // MON-12: info and above, never raw I/O; MON-42: the subject if !typ.is_raw_io() - && (is_end || event.inner.level as usize >= Level::Info as usize) + && (is_end + || event.inner.level as usize >= Level::Info as usize + || typ == EventType::Queue(QueueEvent::MessageSubject)) { if span.events.len() < MAX_EVENTS - 1 || is_end { span.events.push(event); diff --git a/crates/jmap/src/inbuxa/telemetry.rs b/crates/jmap/src/inbuxa/telemetry.rs index 2c2f0286..cc22d9f3 100644 --- a/crates/jmap/src/inbuxa/telemetry.rs +++ b/crates/jmap/src/inbuxa/telemetry.rs @@ -234,11 +234,13 @@ use store::{ use trc::{AddContext, EventType, Key}; /// The trace's server-set fields (MON-14): the first event's time, the -/// first `from`, every distinct `to`, and the message size or 0. +/// first `from`, every distinct `to`, the message size or 0, and the first +/// `subject` (MON-42). fn trace_to_value(id: u64, trace: Trace) -> JmapValue<'static> { let mut from = None; let mut to: Vec = Vec::new(); let mut size = None; + let mut subject = None; let mut first_timestamp = None; for event in trace.events.iter() { first_timestamp.get_or_insert(event.timestamp.timestamp()); @@ -264,6 +266,11 @@ fn trace_to_value(id: u64, trace: Trace) -> JmapValue<'static> { from = texts.into_iter().next(); } } + Key::Subject => { + if subject.is_none() { + subject = texts.into_iter().next(); + } + } Key::To => { for text in texts { if !to.contains(&text) { @@ -292,6 +299,13 @@ fn trace_to_value(id: u64, trace: Trace) -> JmapValue<'static> { ); obj.insert_unchecked(Property::To, JmapValue::Str(to.join(", ").into())); obj.insert_unchecked(Property::Size, JmapValue::Number(size.unwrap_or(0).into())); + obj.insert_unchecked( + Property::Subject, + match subject { + Some(subject) => JmapValue::Str(subject.into()), + None => JmapValue::Null, + }, + ); } value } diff --git a/crates/smtp/src/outbound/delivery.rs b/crates/smtp/src/outbound/delivery.rs index 8600e46a..6fb6b48f 100644 --- a/crates/smtp/src/outbound/delivery.rs +++ b/crates/smtp/src/outbound/delivery.rs @@ -57,6 +57,20 @@ impl QueuedMessage { message.span_id = server.inner.data.span_id_gen.generate(); let span_id = message.span_id; + // inbuxa: the subject for delivery history (#228), read + // only when history is kept; a fresh message comes from + // this node's blob cache, and delivery reads it anyway + let subject = if !server.tracing_store().is_none() { + server + .get_message_blob(message.message.blob_hash.as_slice(), 0..usize::MAX) + .await + .ok() + .flatten() + .and_then(|blob| crate::queue::message_subject(&blob)) + } else { + None + }; + trc::event!( Delivery(DeliveryEvent::AttemptStart), SpanId = message.span_id, @@ -86,6 +100,15 @@ impl QueuedMessage { Size = message.message.size, Total = message.message.recipients.len(), ); + // inbuxa: inbuxa-admin #228 + if let Some(subject) = subject { + trc::event!( + Queue(trc::QueueEvent::MessageSubject), + SpanId = span_id, + QueueId = message.queue_id, + Subject = subject, + ); + } // Attempt delivery let start_time = Instant::now(); diff --git a/crates/smtp/src/queue/mod.rs b/crates/smtp/src/queue/mod.rs index 4f6a37da..a6eae2b9 100644 --- a/crates/smtp/src/queue/mod.rs +++ b/crates/smtp/src/queue/mod.rs @@ -87,6 +87,23 @@ pub struct Message { pub metadata: Box<[Metadata]>, } +/// inbuxa: the longest subject kept in delivery history (#228) +const SUBJECT_MAX_CHARS: usize = 255; + +/// inbuxa: a message's subject for its delivery history (#228), from its +/// headers alone, cut to `SUBJECT_MAX_CHARS`. `None` when it has none. +pub fn message_subject(raw_message: &[u8]) -> Option { + let parsed = mail_parser::MessageParser::new().parse_headers(raw_message)?; + let subject = parsed.subject()?.trim(); + if subject.is_empty() { + return None; + } + Some(match subject.char_indices().nth(SUBJECT_MAX_CHARS) { + Some((end, _)) => format!("{}…", &subject[..end]), + None => subject.to_string(), + }) +} + impl Message { pub fn queued_event(&self) -> trc::QueueEvent { if (self.flags & FROM_AUTHENTICATED) != 0 { @@ -747,3 +764,28 @@ mod smtp_path_tests { } } } + +#[cfg(test)] +mod subject_tests { + use super::message_subject; + + #[test] + fn subject_from_headers() { + assert_eq!( + message_subject( + b"From: a@example.org\r\nSubject: =?utf-8?q?Caf=C3=A9_menu?=\r\n\r\nbody" + ) + .as_deref(), + Some("Café menu") + ); + assert_eq!( + message_subject(b"From: a@example.org\r\n\r\nSubject: not a header"), + None + ); + assert_eq!(message_subject(b"Subject: \r\n\r\n"), None); + let long = format!("Subject: {}\r\n\r\n", "é".repeat(300)); + let subject = message_subject(long.as_bytes()).unwrap(); + assert_eq!(subject.chars().count(), 256); + assert!(subject.ends_with('…')); + } +} diff --git a/crates/smtp/src/queue/spool.rs b/crates/smtp/src/queue/spool.rs index da8d9a4f..582987e0 100644 --- a/crates/smtp/src/queue/spool.rs +++ b/crates/smtp/src/queue/spool.rs @@ -396,6 +396,13 @@ impl MessageWrapper { .. } = params; let event = self.message.queued_event(); + // inbuxa: inbuxa-admin #228: the subject, only while delivery + // history is kept + let subject = if !server.tracing_store().is_none() { + crate::queue::message_subject(raw_message) + } else { + None + }; // Write blob let raw_headers = raw_headers.unwrap_or_default(); @@ -503,6 +510,15 @@ impl MessageWrapper { NextDsn = self.message.next_dsn(None).map(trc::Value::Timestamp), Expires = self.message.expires(None).map(trc::Value::Timestamp), ); + // inbuxa: inbuxa-admin #228: the subject for delivery history + if let Some(subject) = subject { + trc::event!( + Queue(trc::QueueEvent::MessageSubject), + SpanId = session_id, + QueueId = self.queue_id, + Subject = subject, + ); + } // Write message to queue let mut batch = BatchBuilder::new(); diff --git a/crates/trc/src/event/enums.rs b/crates/trc/src/event/enums.rs index 2879a354..494e0dec 100644 --- a/crates/trc/src/event/enums.rs +++ b/crates/trc/src/event/enums.rs @@ -15,10 +15,10 @@ // (audit-hold-lock spec, AU-3, AU-8); 649 is cluster.node-id-reassigned; // 650 to 659 are inbuxa Relay's (relay spec, RL-23, RL-T23); 660 is // store.blob-read-through; 661 is smtp.directory-no-answer (#364); 662 is -// spam.result (#397). +// spam.result (#397); 663 is queue.message-subject (inbuxa-admin #228). // TOTAL_EVENT_COUNT sizes the collector's per-event tables, so it is one // more than the highest id; the test below holds it to the variant list -pub const TOTAL_EVENT_COUNT: usize = 663; +pub const TOTAL_EVENT_COUNT: usize = 664; pub const TOTAL_METRIC_COUNT: usize = 369; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -608,6 +608,9 @@ pub enum PushSubscriptionEvent { pub enum QueueEvent { Started = 390, MessageQueued = 380, + /// inbuxa: inbuxa-admin #228: a queued message's subject, for delivery + /// history. Debug level, so an info log never holds subjects. + MessageSubject = 663, AuthenticatedMessageQueued = 381, ReportQueued = 382, DsnQueued = 379, @@ -1418,6 +1421,8 @@ pub enum Key { Value = 63, Version = 64, QueueName = 65, + // inbuxa: a message's subject in delivery history (#228) + Subject = 66, } #[cfg(test)] diff --git a/crates/trc/src/event/enums_impl.rs b/crates/trc/src/event/enums_impl.rs index 63187118..aa324ec5 100644 --- a/crates/trc/src/event/enums_impl.rs +++ b/crates/trc/src/event/enums_impl.rs @@ -426,6 +426,8 @@ impl EventType { b"push-subscription.not-found" => EventType::PushSubscription(PushSubscriptionEvent::NotFound), b"queue.started" => EventType::Queue(QueueEvent::Started), b"queue.message-queued" => EventType::Queue(QueueEvent::MessageQueued), + // inbuxa: inbuxa-admin #228 + b"queue.message-subject" => EventType::Queue(QueueEvent::MessageSubject), b"queue.authenticated-message-queued" => EventType::Queue(QueueEvent::AuthenticatedMessageQueued), b"queue.report-queued" => EventType::Queue(QueueEvent::ReportQueued), b"queue.dsn-queued" => EventType::Queue(QueueEvent::DsnQueued), @@ -1211,6 +1213,8 @@ impl EventType { } EventType::Queue(QueueEvent::Started) => "queue.started", EventType::Queue(QueueEvent::MessageQueued) => "queue.message-queued", + // inbuxa: inbuxa-admin #228 + EventType::Queue(QueueEvent::MessageSubject) => "queue.message-subject", EventType::Queue(QueueEvent::AuthenticatedMessageQueued) => { "queue.authenticated-message-queued" } @@ -1921,6 +1925,8 @@ impl EventType { EventType::PushSubscription(PushSubscriptionEvent::NotFound) => 372, EventType::Queue(QueueEvent::Started) => 390, EventType::Queue(QueueEvent::MessageQueued) => 380, + // inbuxa: inbuxa-admin #228 + EventType::Queue(QueueEvent::MessageSubject) => 663, EventType::Queue(QueueEvent::AuthenticatedMessageQueued) => 381, EventType::Queue(QueueEvent::ReportQueued) => 382, EventType::Queue(QueueEvent::DsnQueued) => 379, @@ -2635,6 +2641,8 @@ impl EventType { 372 => Some(EventType::PushSubscription(PushSubscriptionEvent::NotFound)), 390 => Some(EventType::Queue(QueueEvent::Started)), 380 => Some(EventType::Queue(QueueEvent::MessageQueued)), + // inbuxa: inbuxa-admin #228 + 663 => Some(EventType::Queue(QueueEvent::MessageSubject)), 381 => Some(EventType::Queue(QueueEvent::AuthenticatedMessageQueued)), 382 => Some(EventType::Queue(QueueEvent::ReportQueued)), 379 => Some(EventType::Queue(QueueEvent::DsnQueued)), @@ -3081,6 +3089,8 @@ impl EventType { EventType::OutgoingReport(OutgoingReportEvent::Locked) => Level::Info, EventType::Queue(QueueEvent::Started) => Level::Info, EventType::Queue(QueueEvent::MessageQueued) => Level::Info, + // inbuxa: inbuxa-admin #228: debug, so info logs never hold subjects + EventType::Queue(QueueEvent::MessageSubject) => Level::Debug, EventType::Queue(QueueEvent::AuthenticatedMessageQueued) => Level::Info, EventType::Queue(QueueEvent::ReportQueued) => Level::Info, EventType::Queue(QueueEvent::DsnQueued) => Level::Info, @@ -3816,6 +3826,8 @@ impl EventType { } EventType::Queue(QueueEvent::Started) => "MTA queue started", EventType::Queue(QueueEvent::MessageQueued) => "Queued message for delivery", + // inbuxa: inbuxa-admin #228 + EventType::Queue(QueueEvent::MessageSubject) => "Subject of the queued message", EventType::Queue(QueueEvent::AuthenticatedMessageQueued) => { "Queued message submission for delivery" } @@ -4830,6 +4842,8 @@ impl EventType { EventType::PushSubscription(PushSubscriptionEvent::NotFound), EventType::Queue(QueueEvent::Started), EventType::Queue(QueueEvent::MessageQueued), + // inbuxa: inbuxa-admin #228 + EventType::Queue(QueueEvent::MessageSubject), EventType::Queue(QueueEvent::AuthenticatedMessageQueued), EventType::Queue(QueueEvent::ReportQueued), EventType::Queue(QueueEvent::DsnQueued), @@ -8162,6 +8176,7 @@ impl Key { b"value" => Key::Value, b"version" => Key::Version, b"queueName" => Key::QueueName, + b"subject" => Key::Subject, } .copied() } @@ -8234,6 +8249,7 @@ impl Key { Key::Value => "value", Key::Version => "version", Key::QueueName => "queueName", + Key::Subject => "subject", } } @@ -8309,11 +8325,12 @@ impl Key { 63 => Some(Key::Value), 64 => Some(Key::Version), 65 => Some(Key::QueueName), + 66 => Some(Key::Subject), _ => None, } } - pub const COUNT: usize = 66; + pub const COUNT: usize = 67; } impl serde::Serialize for Key { diff --git a/docs/spec/features/monitoring.md b/docs/spec/features/monitoring.md index 05244177..1dc7c777 100644 --- a/docs/spec/features/monitoring.md +++ b/docs/spec/features/monitoring.md @@ -77,7 +77,7 @@ Clean-up of expired history runs on the existing `dataCleanupSchedule`. `timestamp`, and `keyValues`: a list of `key` (a `Key`) and `value` (a `TraceValue`: `String`, `UnsignedInt`, `Integer`, `Boolean`, `Float`, `UTCDateTime`, `Duration`, `IpAddr`, `List`, `Event`, `Null`). Server-set: -`timestamp`, `from`, `to` (a string), `size`. Views: `x:Trace/InboundDelivery` +`timestamp`, `from`, `to` (a string), `subject` (MON-42), `size`. Views: `x:Trace/InboundDelivery` and `x:Trace/OutboundDelivery`. ### The metric sample, `x:Metric` (read-only) @@ -240,6 +240,18 @@ unchanged. with its points, highest first (X-Spam-Result lists them lowest first). When one rule rejects or discards a message by itself, `Details` names only that rule. An inbound session's trace therefore shows why its message scored the way it did. +- **MON-42.** (inbuxa-admin #228) A trace's `subject` is the first `subject` + value in its events, or null. While a tracing store is set, the server + raises `queue.message-subject` (debug level, id 663) with the message's + `Subject` header, cut to 255 characters: in the SMTP session when it + queues a message, and in the delivery attempt right after + `delivery.attempt-start`, read from the message's blob (a fresh message + comes from the node's blob cache). The trace history keeps this one debug + event, so an info-level log or OpenTelemetry tracer never holds subjects; + a tracer set to debug does. With no tracing store nothing is read or + raised. Both list views show it between To and Size. It isn't in the + `text` search keywords. Like every other trace field it is kept for + `holdTracesFor`, and only the superuser set can read it. ### Retention and growth @@ -567,7 +579,7 @@ they cover tests 10 and 12 (`tracing.rs`), 13 (`metrics.rs`) and 18 (`alerts.rs`). Tests 1, 3, 5 to 9, 11, 14 to 17, 19 to 21, 23 and 24 pass as `tests/src/system/monitoring.rs`. -- **MON-1 to MON-41:** built, with the limits below. +- **MON-1 to MON-42:** built, with the limits below. - **Test 26 (compat)** is written as `monitoring_compat`, ignored, and unrun until a copy of INBUXA's data is provided. It checks observed 1, reads the old history without an error, and purges it. It deletes history, so only diff --git a/docs/spec/features/personal-data-catalog.md b/docs/spec/features/personal-data-catalog.md index fc0224ea..f2903e42 100644 --- a/docs/spec/features/personal-data-catalog.md +++ b/docs/spec/features/personal-data-catalog.md @@ -463,7 +463,7 @@ Response `list[0]`: { "id": "x:Trace", "kind": "object", - "categories": ["network", "identifier", "metadata"], + "categories": ["network", "identifier", "metadata", "content"], "collected": true, "retention": { "days": 30, "setting": "x:DataRetention.holdTracesFor" }, "leavesHost": false, diff --git a/resources/privacy/catalog.toml b/resources/privacy/catalog.toml index 1b052612..92fde3db 100644 --- a/resources/privacy/catalog.toml +++ b/resources/privacy/catalog.toml @@ -2268,9 +2268,11 @@ where = ["data-store", "search-store"] scope = "server" retention = { setting = "x:DataRetention.holdTracesFor" } [object."x:Trace".properties] -events = ["network", "identifier", "metadata"] +events = ["network", "identifier", "metadata", "content"] from = ["identifier"] size = ["metadata"] +# inbuxa-admin #228: the message's Subject header +subject = ["content"] timestamp = ["metadata"] to = ["identifier"] @@ -2281,7 +2283,7 @@ where = ["data-store"] scope = "server" retention = { setting = "x:DataRetention.holdTracesFor" } [object."x:TraceEvent".properties] -keyValues = ["network", "identifier", "metadata"] +keyValues = ["network", "identifier", "metadata", "content"] timestamp = ["metadata"] [object."x:TraceKeyValue"] @@ -2291,7 +2293,7 @@ where = ["data-store"] scope = "server" retention = { setting = "x:DataRetention.holdTracesFor" } [object."x:TraceKeyValue".properties] -value = ["network", "identifier", "metadata"] +value = ["network", "identifier", "metadata", "content"] [object."x:TraceValueBoolean"] default = "none" diff --git a/resources/schema/schema.json.gz b/resources/schema/schema.json.gz index 2a65607d..f3098ce5 100644 Binary files a/resources/schema/schema.json.gz and b/resources/schema/schema.json.gz differ diff --git a/resources/schema/schema.json.sha256 b/resources/schema/schema.json.sha256 index 345ea456..589e80a4 100644 --- a/resources/schema/schema.json.sha256 +++ b/resources/schema/schema.json.sha256 @@ -1 +1 @@ -UDW2s5eJMuFvg1FJkP9HP4a1IApTtrR0Vj9awc3VjhY \ No newline at end of file +WWnyz0UpWEvf1aqR3HadgQyWuWDvGtqQXwpOgmnafSg \ No newline at end of file diff --git a/tests/src/system/monitoring.rs b/tests/src/system/monitoring.rs index df92163f..0383c7af 100644 --- a/tests/src/system/monitoring.rs +++ b/tests/src/system/monitoring.rs @@ -167,6 +167,11 @@ pub async fn test(test: &mut TestServer) { .is_some_and(|to| to.contains("mon@example.org"))), "test 9: to" ); + // #228: the session and the delivery attempt both name the subject + assert!( + traces.iter().all(|t| t["subject"] == "Traced"), + "test 9: subject, {traces:?}" + ); // #397: the session came from outside and the attempt only filed the // message here, so each list's default leaves out the other kind