Delivery history: show each message's subject #452
No files matched your search
@@ -516,7 +516,9 @@ impl Tracers {
|
||||
&& (Level::Info.is_contained(event_level)
|
||||
|| event_type.is_span_start()
|
||||
|| event_type.is_span_end()
|
||||
|| event_type.as_str().starts_with("smtp.mail-from"))
|
||||
|| event_type.as_str().starts_with("smtp.mail-from")
|
||||
// inbuxa-admin #228: debug, so it reaches no info log
|
||||
|| *event_type == EventType::Queue(trc::QueueEvent::MessageSubject))
|
||||
{
|
||||
interests.set(event_type.to_id() as usize);
|
||||
global_interests.set(event_type.to_id() as usize);
|
||||
|
||||
@@ -28,8 +28,8 @@ use store::{
|
||||
write::{BatchBuilder, SearchIndex, TelemetryClass, ValueClass, now},
|
||||
};
|
||||
use trc::{
|
||||
AddContext, AuthEvent, DeliveryEvent, Event, EventDetails, EventType, Key, Level, SmtpEvent,
|
||||
ipc::subscriber::SubscriberBuilder,
|
||||
AddContext, AuthEvent, DeliveryEvent, Event, EventDetails, EventType, Key, Level, QueueEvent,
|
||||
SmtpEvent, ipc::subscriber::SubscriberBuilder,
|
||||
};
|
||||
use utils::snowflake::SnowflakeIdGenerator;
|
||||
|
||||
@@ -270,9 +270,11 @@ pub(crate) fn spawn_store_tracer(builder: SubscriberBuilder, tracing: Store, dat
|
||||
span.has_mail_from = true;
|
||||
}
|
||||
let is_end = typ.is_span_end();
|
||||
// MON-12: info and above, never raw I/O
|
||||
// MON-12: info and above, never raw I/O; MON-42: the subject
|
||||
if !typ.is_raw_io()
|
||||
&& (is_end || event.inner.level as usize >= Level::Info as usize)
|
||||
&& (is_end
|
||||
|| event.inner.level as usize >= Level::Info as usize
|
||||
|| typ == EventType::Queue(QueueEvent::MessageSubject))
|
||||
{
|
||||
if span.events.len() < MAX_EVENTS - 1 || is_end {
|
||||
span.events.push(event);
|
||||
|
||||
@@ -234,11 +234,13 @@ use store::{
|
||||
use trc::{AddContext, EventType, Key};
|
||||
|
||||
/// The trace's server-set fields (MON-14): the first event's time, the
|
||||
/// first `from`, every distinct `to`, and the message size or 0.
|
||||
/// first `from`, every distinct `to`, the message size or 0, and the first
|
||||
/// `subject` (MON-42).
|
||||
fn trace_to_value(id: u64, trace: Trace) -> JmapValue<'static> {
|
||||
let mut from = None;
|
||||
let mut to: Vec<String> = Vec::new();
|
||||
let mut size = None;
|
||||
let mut subject = None;
|
||||
let mut first_timestamp = None;
|
||||
for event in trace.events.iter() {
|
||||
first_timestamp.get_or_insert(event.timestamp.timestamp());
|
||||
@@ -264,6 +266,11 @@ fn trace_to_value(id: u64, trace: Trace) -> JmapValue<'static> {
|
||||
from = texts.into_iter().next();
|
||||
}
|
||||
}
|
||||
Key::Subject => {
|
||||
if subject.is_none() {
|
||||
subject = texts.into_iter().next();
|
||||
}
|
||||
}
|
||||
Key::To => {
|
||||
for text in texts {
|
||||
if !to.contains(&text) {
|
||||
@@ -292,6 +299,13 @@ fn trace_to_value(id: u64, trace: Trace) -> JmapValue<'static> {
|
||||
);
|
||||
obj.insert_unchecked(Property::To, JmapValue::Str(to.join(", ").into()));
|
||||
obj.insert_unchecked(Property::Size, JmapValue::Number(size.unwrap_or(0).into()));
|
||||
obj.insert_unchecked(
|
||||
Property::Subject,
|
||||
match subject {
|
||||
Some(subject) => JmapValue::Str(subject.into()),
|
||||
None => JmapValue::Null,
|
||||
},
|
||||
);
|
||||
}
|
||||
value
|
||||
}
|
||||
|
||||
@@ -57,6 +57,20 @@ impl QueuedMessage {
|
||||
message.span_id = server.inner.data.span_id_gen.generate();
|
||||
let span_id = message.span_id;
|
||||
|
||||
// inbuxa: the subject for delivery history (#228), read
|
||||
// only when history is kept; a fresh message comes from
|
||||
// this node's blob cache, and delivery reads it anyway
|
||||
let subject = if !server.tracing_store().is_none() {
|
||||
server
|
||||
.get_message_blob(message.message.blob_hash.as_slice(), 0..usize::MAX)
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
.and_then(|blob| crate::queue::message_subject(&blob))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
trc::event!(
|
||||
Delivery(DeliveryEvent::AttemptStart),
|
||||
SpanId = message.span_id,
|
||||
@@ -86,6 +100,15 @@ impl QueuedMessage {
|
||||
Size = message.message.size,
|
||||
Total = message.message.recipients.len(),
|
||||
);
|
||||
// inbuxa: inbuxa-admin #228
|
||||
if let Some(subject) = subject {
|
||||
trc::event!(
|
||||
Queue(trc::QueueEvent::MessageSubject),
|
||||
SpanId = span_id,
|
||||
QueueId = message.queue_id,
|
||||
Subject = subject,
|
||||
);
|
||||
}
|
||||
|
||||
// Attempt delivery
|
||||
let start_time = Instant::now();
|
||||
|
||||
@@ -87,6 +87,23 @@ pub struct Message {
|
||||
pub metadata: Box<[Metadata]>,
|
||||
}
|
||||
|
||||
/// inbuxa: the longest subject kept in delivery history (#228)
|
||||
const SUBJECT_MAX_CHARS: usize = 255;
|
||||
|
||||
/// inbuxa: a message's subject for its delivery history (#228), from its
|
||||
/// headers alone, cut to `SUBJECT_MAX_CHARS`. `None` when it has none.
|
||||
pub fn message_subject(raw_message: &[u8]) -> Option<String> {
|
||||
let parsed = mail_parser::MessageParser::new().parse_headers(raw_message)?;
|
||||
let subject = parsed.subject()?.trim();
|
||||
if subject.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(match subject.char_indices().nth(SUBJECT_MAX_CHARS) {
|
||||
Some((end, _)) => format!("{}…", &subject[..end]),
|
||||
None => subject.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
impl Message {
|
||||
pub fn queued_event(&self) -> trc::QueueEvent {
|
||||
if (self.flags & FROM_AUTHENTICATED) != 0 {
|
||||
@@ -747,3 +764,28 @@ mod smtp_path_tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod subject_tests {
|
||||
use super::message_subject;
|
||||
|
||||
#[test]
|
||||
fn subject_from_headers() {
|
||||
assert_eq!(
|
||||
message_subject(
|
||||
b"From: [email protected]\r\nSubject: =?utf-8?q?Caf=C3=A9_menu?=\r\n\r\nbody"
|
||||
)
|
||||
.as_deref(),
|
||||
Some("Café menu")
|
||||
);
|
||||
assert_eq!(
|
||||
message_subject(b"From: [email protected]\r\n\r\nSubject: not a header"),
|
||||
None
|
||||
);
|
||||
assert_eq!(message_subject(b"Subject: \r\n\r\n"), None);
|
||||
let long = format!("Subject: {}\r\n\r\n", "é".repeat(300));
|
||||
let subject = message_subject(long.as_bytes()).unwrap();
|
||||
assert_eq!(subject.chars().count(), 256);
|
||||
assert!(subject.ends_with('…'));
|
||||
}
|
||||
}
|
||||
@@ -396,6 +396,13 @@ impl MessageWrapper {
|
||||
..
|
||||
} = params;
|
||||
let event = self.message.queued_event();
|
||||
// inbuxa: inbuxa-admin #228: the subject, only while delivery
|
||||
// history is kept
|
||||
let subject = if !server.tracing_store().is_none() {
|
||||
crate::queue::message_subject(raw_message)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Write blob
|
||||
let raw_headers = raw_headers.unwrap_or_default();
|
||||
@@ -503,6 +510,15 @@ impl MessageWrapper {
|
||||
NextDsn = self.message.next_dsn(None).map(trc::Value::Timestamp),
|
||||
Expires = self.message.expires(None).map(trc::Value::Timestamp),
|
||||
);
|
||||
// inbuxa: inbuxa-admin #228: the subject for delivery history
|
||||
if let Some(subject) = subject {
|
||||
trc::event!(
|
||||
Queue(trc::QueueEvent::MessageSubject),
|
||||
SpanId = session_id,
|
||||
QueueId = self.queue_id,
|
||||
Subject = subject,
|
||||
);
|
||||
}
|
||||
|
||||
// Write message to queue
|
||||
let mut batch = BatchBuilder::new();
|
||||
|
||||
@@ -15,10 +15,10 @@
|
||||
// (audit-hold-lock spec, AU-3, AU-8); 649 is cluster.node-id-reassigned;
|
||||
// 650 to 659 are inbuxa Relay's (relay spec, RL-23, RL-T23); 660 is
|
||||
// store.blob-read-through; 661 is smtp.directory-no-answer (#364); 662 is
|
||||
// spam.result (#397).
|
||||
// spam.result (#397); 663 is queue.message-subject (inbuxa-admin #228).
|
||||
// TOTAL_EVENT_COUNT sizes the collector's per-event tables, so it is one
|
||||
// more than the highest id; the test below holds it to the variant list
|
||||
pub const TOTAL_EVENT_COUNT: usize = 663;
|
||||
pub const TOTAL_EVENT_COUNT: usize = 664;
|
||||
pub const TOTAL_METRIC_COUNT: usize = 369;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
@@ -608,6 +608,9 @@ pub enum PushSubscriptionEvent {
|
||||
pub enum QueueEvent {
|
||||
Started = 390,
|
||||
MessageQueued = 380,
|
||||
/// inbuxa: inbuxa-admin #228: a queued message's subject, for delivery
|
||||
/// history. Debug level, so an info log never holds subjects.
|
||||
MessageSubject = 663,
|
||||
AuthenticatedMessageQueued = 381,
|
||||
ReportQueued = 382,
|
||||
DsnQueued = 379,
|
||||
@@ -1418,6 +1421,8 @@ pub enum Key {
|
||||
Value = 63,
|
||||
Version = 64,
|
||||
QueueName = 65,
|
||||
// inbuxa: a message's subject in delivery history (#228)
|
||||
Subject = 66,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -426,6 +426,8 @@ impl EventType {
|
||||
b"push-subscription.not-found" => EventType::PushSubscription(PushSubscriptionEvent::NotFound),
|
||||
b"queue.started" => EventType::Queue(QueueEvent::Started),
|
||||
b"queue.message-queued" => EventType::Queue(QueueEvent::MessageQueued),
|
||||
// inbuxa: inbuxa-admin #228
|
||||
b"queue.message-subject" => EventType::Queue(QueueEvent::MessageSubject),
|
||||
b"queue.authenticated-message-queued" => EventType::Queue(QueueEvent::AuthenticatedMessageQueued),
|
||||
b"queue.report-queued" => EventType::Queue(QueueEvent::ReportQueued),
|
||||
b"queue.dsn-queued" => EventType::Queue(QueueEvent::DsnQueued),
|
||||
@@ -1211,6 +1213,8 @@ impl EventType {
|
||||
}
|
||||
EventType::Queue(QueueEvent::Started) => "queue.started",
|
||||
EventType::Queue(QueueEvent::MessageQueued) => "queue.message-queued",
|
||||
// inbuxa: inbuxa-admin #228
|
||||
EventType::Queue(QueueEvent::MessageSubject) => "queue.message-subject",
|
||||
EventType::Queue(QueueEvent::AuthenticatedMessageQueued) => {
|
||||
"queue.authenticated-message-queued"
|
||||
}
|
||||
@@ -1921,6 +1925,8 @@ impl EventType {
|
||||
EventType::PushSubscription(PushSubscriptionEvent::NotFound) => 372,
|
||||
EventType::Queue(QueueEvent::Started) => 390,
|
||||
EventType::Queue(QueueEvent::MessageQueued) => 380,
|
||||
// inbuxa: inbuxa-admin #228
|
||||
EventType::Queue(QueueEvent::MessageSubject) => 663,
|
||||
EventType::Queue(QueueEvent::AuthenticatedMessageQueued) => 381,
|
||||
EventType::Queue(QueueEvent::ReportQueued) => 382,
|
||||
EventType::Queue(QueueEvent::DsnQueued) => 379,
|
||||
@@ -2635,6 +2641,8 @@ impl EventType {
|
||||
372 => Some(EventType::PushSubscription(PushSubscriptionEvent::NotFound)),
|
||||
390 => Some(EventType::Queue(QueueEvent::Started)),
|
||||
380 => Some(EventType::Queue(QueueEvent::MessageQueued)),
|
||||
// inbuxa: inbuxa-admin #228
|
||||
663 => Some(EventType::Queue(QueueEvent::MessageSubject)),
|
||||
381 => Some(EventType::Queue(QueueEvent::AuthenticatedMessageQueued)),
|
||||
382 => Some(EventType::Queue(QueueEvent::ReportQueued)),
|
||||
379 => Some(EventType::Queue(QueueEvent::DsnQueued)),
|
||||
@@ -3081,6 +3089,8 @@ impl EventType {
|
||||
EventType::OutgoingReport(OutgoingReportEvent::Locked) => Level::Info,
|
||||
EventType::Queue(QueueEvent::Started) => Level::Info,
|
||||
EventType::Queue(QueueEvent::MessageQueued) => Level::Info,
|
||||
// inbuxa: inbuxa-admin #228: debug, so info logs never hold subjects
|
||||
EventType::Queue(QueueEvent::MessageSubject) => Level::Debug,
|
||||
EventType::Queue(QueueEvent::AuthenticatedMessageQueued) => Level::Info,
|
||||
EventType::Queue(QueueEvent::ReportQueued) => Level::Info,
|
||||
EventType::Queue(QueueEvent::DsnQueued) => Level::Info,
|
||||
@@ -3816,6 +3826,8 @@ impl EventType {
|
||||
}
|
||||
EventType::Queue(QueueEvent::Started) => "MTA queue started",
|
||||
EventType::Queue(QueueEvent::MessageQueued) => "Queued message for delivery",
|
||||
// inbuxa: inbuxa-admin #228
|
||||
EventType::Queue(QueueEvent::MessageSubject) => "Subject of the queued message",
|
||||
EventType::Queue(QueueEvent::AuthenticatedMessageQueued) => {
|
||||
"Queued message submission for delivery"
|
||||
}
|
||||
@@ -4830,6 +4842,8 @@ impl EventType {
|
||||
EventType::PushSubscription(PushSubscriptionEvent::NotFound),
|
||||
EventType::Queue(QueueEvent::Started),
|
||||
EventType::Queue(QueueEvent::MessageQueued),
|
||||
// inbuxa: inbuxa-admin #228
|
||||
EventType::Queue(QueueEvent::MessageSubject),
|
||||
EventType::Queue(QueueEvent::AuthenticatedMessageQueued),
|
||||
EventType::Queue(QueueEvent::ReportQueued),
|
||||
EventType::Queue(QueueEvent::DsnQueued),
|
||||
@@ -8162,6 +8176,7 @@ impl Key {
|
||||
b"value" => Key::Value,
|
||||
b"version" => Key::Version,
|
||||
b"queueName" => Key::QueueName,
|
||||
b"subject" => Key::Subject,
|
||||
}
|
||||
.copied()
|
||||
}
|
||||
@@ -8234,6 +8249,7 @@ impl Key {
|
||||
Key::Value => "value",
|
||||
Key::Version => "version",
|
||||
Key::QueueName => "queueName",
|
||||
Key::Subject => "subject",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8309,11 +8325,12 @@ impl Key {
|
||||
63 => Some(Key::Value),
|
||||
64 => Some(Key::Version),
|
||||
65 => Some(Key::QueueName),
|
||||
66 => Some(Key::Subject),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub const COUNT: usize = 66;
|
||||
pub const COUNT: usize = 67;
|
||||
}
|
||||
|
||||
impl serde::Serialize for Key {
|
||||
|
||||
@@ -77,7 +77,7 @@ Clean-up of expired history runs on the existing `dataCleanupSchedule`.
|
||||
`timestamp`, and `keyValues`: a list of `key` (a `Key`) and `value` (a
|
||||
`TraceValue`: `String`, `UnsignedInt`, `Integer`, `Boolean`, `Float`,
|
||||
`UTCDateTime`, `Duration`, `IpAddr`, `List`, `Event`, `Null`). Server-set:
|
||||
`timestamp`, `from`, `to` (a string), `size`. Views: `x:Trace/InboundDelivery`
|
||||
`timestamp`, `from`, `to` (a string), `subject` (MON-42), `size`. Views: `x:Trace/InboundDelivery`
|
||||
and `x:Trace/OutboundDelivery`.
|
||||
|
||||
### The metric sample, `x:Metric` (read-only)
|
||||
@@ -240,6 +240,18 @@ unchanged.
|
||||
with its points, highest first (X-Spam-Result lists them lowest first). When one rule rejects or
|
||||
discards a message by itself, `Details` names only that rule. An inbound
|
||||
session's trace therefore shows why its message scored the way it did.
|
||||
- **MON-42.** (inbuxa-admin #228) A trace's `subject` is the first `subject`
|
||||
value in its events, or null. While a tracing store is set, the server
|
||||
raises `queue.message-subject` (debug level, id 663) with the message's
|
||||
`Subject` header, cut to 255 characters: in the SMTP session when it
|
||||
queues a message, and in the delivery attempt right after
|
||||
`delivery.attempt-start`, read from the message's blob (a fresh message
|
||||
comes from the node's blob cache). The trace history keeps this one debug
|
||||
event, so an info-level log or OpenTelemetry tracer never holds subjects;
|
||||
a tracer set to debug does. With no tracing store nothing is read or
|
||||
raised. Both list views show it between To and Size. It isn't in the
|
||||
`text` search keywords. Like every other trace field it is kept for
|
||||
`holdTracesFor`, and only the superuser set can read it.
|
||||
|
||||
### Retention and growth
|
||||
|
||||
@@ -567,7 +579,7 @@ they cover tests 10 and 12 (`tracing.rs`), 13 (`metrics.rs`) and 18
|
||||
(`alerts.rs`). Tests 1, 3, 5 to 9, 11, 14 to 17, 19 to 21, 23 and 24 pass as
|
||||
`tests/src/system/monitoring.rs`.
|
||||
|
||||
- **MON-1 to MON-41:** built, with the limits below.
|
||||
- **MON-1 to MON-42:** built, with the limits below.
|
||||
- **Test 26 (compat)** is written as `monitoring_compat`, ignored, and unrun
|
||||
until a copy of INBUXA's data is provided. It checks observed 1, reads the
|
||||
old history without an error, and purges it. It deletes history, so only
|
||||
|
||||
@@ -463,7 +463,7 @@ Response `list[0]`:
|
||||
{
|
||||
"id": "x:Trace",
|
||||
"kind": "object",
|
||||
"categories": ["network", "identifier", "metadata"],
|
||||
"categories": ["network", "identifier", "metadata", "content"],
|
||||
"collected": true,
|
||||
"retention": { "days": 30, "setting": "x:DataRetention.holdTracesFor" },
|
||||
"leavesHost": false,
|
||||
|
||||
@@ -2268,9 +2268,11 @@ where = ["data-store", "search-store"]
|
||||
scope = "server"
|
||||
retention = { setting = "x:DataRetention.holdTracesFor" }
|
||||
[object."x:Trace".properties]
|
||||
events = ["network", "identifier", "metadata"]
|
||||
events = ["network", "identifier", "metadata", "content"]
|
||||
from = ["identifier"]
|
||||
size = ["metadata"]
|
||||
# inbuxa-admin #228: the message's Subject header
|
||||
subject = ["content"]
|
||||
timestamp = ["metadata"]
|
||||
to = ["identifier"]
|
||||
|
||||
@@ -2281,7 +2283,7 @@ where = ["data-store"]
|
||||
scope = "server"
|
||||
retention = { setting = "x:DataRetention.holdTracesFor" }
|
||||
[object."x:TraceEvent".properties]
|
||||
keyValues = ["network", "identifier", "metadata"]
|
||||
keyValues = ["network", "identifier", "metadata", "content"]
|
||||
timestamp = ["metadata"]
|
||||
|
||||
[object."x:TraceKeyValue"]
|
||||
@@ -2291,7 +2293,7 @@ where = ["data-store"]
|
||||
scope = "server"
|
||||
retention = { setting = "x:DataRetention.holdTracesFor" }
|
||||
[object."x:TraceKeyValue".properties]
|
||||
value = ["network", "identifier", "metadata"]
|
||||
value = ["network", "identifier", "metadata", "content"]
|
||||
|
||||
[object."x:TraceValueBoolean"]
|
||||
default = "none"
|
||||
|
||||
Binary file not shown.
@@ -1 +1 @@
|
||||
UDW2s5eJMuFvg1FJkP9HP4a1IApTtrR0Vj9awc3VjhY
|
||||
WWnyz0UpWEvf1aqR3HadgQyWuWDvGtqQXwpOgmnafSg
|
||||
@@ -167,6 +167,11 @@ pub async fn test(test: &mut TestServer) {
|
||||
.is_some_and(|to| to.contains("[email protected]"))),
|
||||
"test 9: to"
|
||||
);
|
||||
// #228: the session and the delivery attempt both name the subject
|
||||
assert!(
|
||||
traces.iter().all(|t| t["subject"] == "Traced"),
|
||||
"test 9: subject, {traces:?}"
|
||||
);
|
||||
|
||||
// #397: the session came from outside and the attempt only filed the
|
||||
// message here, so each list's default leaves out the other kind
|
||||
|
||||
Reference in new issue
Block a user