Delivery history: scope inbound and outbound by kind; record spam results #398
No files matched your search
@@ -28,7 +28,7 @@ use store::{
|
||||
write::{BatchBuilder, SearchIndex, TelemetryClass, ValueClass, now},
|
||||
};
|
||||
use trc::{
|
||||
AddContext, DeliveryEvent, Event, EventDetails, EventType, Key, Level, SmtpEvent,
|
||||
AddContext, AuthEvent, DeliveryEvent, Event, EventDetails, EventType, Key, Level, SmtpEvent,
|
||||
ipc::subscriber::SubscriberBuilder,
|
||||
};
|
||||
use utils::snowflake::SnowflakeIdGenerator;
|
||||
@@ -84,6 +84,92 @@ pub fn decode_trace(bytes: &[u8]) -> Option<Trace> {
|
||||
})
|
||||
}
|
||||
|
||||
/// What a stored trace records (#397): an SMTP session from another server
|
||||
/// (`received`) or from one of this server's users (`submitted`), and a
|
||||
/// delivery attempt that went to another server (`remote`) or only filed the
|
||||
/// message in local mailboxes (`local`).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum TraceKind {
|
||||
Received,
|
||||
Submitted,
|
||||
Remote,
|
||||
Local,
|
||||
}
|
||||
|
||||
impl TraceKind {
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"received" => Some(TraceKind::Received),
|
||||
"submitted" => Some(TraceKind::Submitted),
|
||||
"remote" => Some(TraceKind::Remote),
|
||||
"local" => Some(TraceKind::Local),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
TraceKind::Received => "received",
|
||||
TraceKind::Submitted => "submitted",
|
||||
TraceKind::Remote => "remote",
|
||||
TraceKind::Local => "local",
|
||||
}
|
||||
}
|
||||
|
||||
/// A trace's kind, from its events; `None` for a trace that opens with
|
||||
/// anything else. Worked out when read, so traces stored before #397
|
||||
/// have one too.
|
||||
pub fn of(trace: &Trace) -> Option<Self> {
|
||||
let mut events = trace.events.iter();
|
||||
match events.next()?.event {
|
||||
EventType::Smtp(SmtpEvent::ConnectionStart) => Some(
|
||||
if events.any(|e| e.event == EventType::Auth(AuthEvent::Success)) {
|
||||
TraceKind::Submitted
|
||||
} else {
|
||||
TraceKind::Received
|
||||
},
|
||||
),
|
||||
EventType::Delivery(DeliveryEvent::AttemptStart) => {
|
||||
// Each route names its type on domain-delivery-start (#397)
|
||||
let mut routes = trace
|
||||
.events
|
||||
.iter()
|
||||
.filter(|e| e.event == EventType::Delivery(DeliveryEvent::DomainDeliveryStart))
|
||||
.filter_map(|e| {
|
||||
e.key_values.iter().find_map(|kv| match &kv.value {
|
||||
TraceValue::String(v) if kv.key == Key::Type => Some(v.value.as_str()),
|
||||
_ => None,
|
||||
})
|
||||
})
|
||||
.peekable();
|
||||
if routes.peek().is_some() {
|
||||
return Some(if routes.all(|route| route == "local") {
|
||||
TraceKind::Local
|
||||
} else {
|
||||
TraceKind::Remote
|
||||
});
|
||||
}
|
||||
// Older traces: local when the message was filed in a
|
||||
// mailbox and no other server was connected to
|
||||
let (mut filed, mut connected) = (false, false);
|
||||
for event in trace.events.iter() {
|
||||
match event.event {
|
||||
EventType::MessageIngest(_) => filed = true,
|
||||
EventType::Delivery(DeliveryEvent::Connect) => connected = true,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Some(if filed && !connected {
|
||||
TraceKind::Local
|
||||
} else {
|
||||
TraceKind::Remote
|
||||
})
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A stored trace as read by key: `None` when it can't be decoded.
|
||||
pub struct MaybeTrace(pub Option<Trace>);
|
||||
|
||||
@@ -230,3 +316,115 @@ pub(crate) fn spawn_store_tracer(builder: SubscriberBuilder, tracing: Store, dat
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use registry::{schema::structs::TraceEvent, types::datetime::UTCDateTime};
|
||||
use trc::MessageIngestEvent;
|
||||
|
||||
fn event(event: EventType, key_values: &[(Key, &str)]) -> TraceEvent {
|
||||
let mut kvs = registry::types::list::List::default();
|
||||
for (key, value) in key_values {
|
||||
kvs.push(TraceKeyValue {
|
||||
key: *key,
|
||||
value: TraceValue::String(TraceValueString {
|
||||
value: value.to_string(),
|
||||
}),
|
||||
});
|
||||
}
|
||||
TraceEvent {
|
||||
event,
|
||||
timestamp: UTCDateTime::now(),
|
||||
key_values: kvs,
|
||||
}
|
||||
}
|
||||
|
||||
fn trace(events: Vec<TraceEvent>) -> Trace {
|
||||
let mut trace = Trace::default();
|
||||
for e in events {
|
||||
trace.events.push(e);
|
||||
}
|
||||
trace
|
||||
}
|
||||
|
||||
const SMTP_START: EventType = EventType::Smtp(SmtpEvent::ConnectionStart);
|
||||
const ATTEMPT: EventType = EventType::Delivery(DeliveryEvent::AttemptStart);
|
||||
const DOMAIN: EventType = EventType::Delivery(DeliveryEvent::DomainDeliveryStart);
|
||||
const CONNECT: EventType = EventType::Delivery(DeliveryEvent::Connect);
|
||||
const HAM: EventType = EventType::MessageIngest(MessageIngestEvent::Ham);
|
||||
|
||||
#[test]
|
||||
fn smtp_sessions() {
|
||||
let received = trace(vec![
|
||||
event(SMTP_START, &[]),
|
||||
event(EventType::Smtp(SmtpEvent::MailFrom), &[]),
|
||||
]);
|
||||
assert_eq!(TraceKind::of(&received), Some(TraceKind::Received));
|
||||
let submitted = trace(vec![
|
||||
event(SMTP_START, &[]),
|
||||
event(EventType::Auth(AuthEvent::Success), &[]),
|
||||
event(EventType::Smtp(SmtpEvent::MailFrom), &[]),
|
||||
]);
|
||||
assert_eq!(TraceKind::of(&submitted), Some(TraceKind::Submitted));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deliveries_by_route() {
|
||||
let local = trace(vec![
|
||||
event(ATTEMPT, &[]),
|
||||
event(
|
||||
DOMAIN,
|
||||
&[(Key::Domain, "example.org"), (Key::Type, "local")],
|
||||
),
|
||||
event(HAM, &[]),
|
||||
]);
|
||||
assert_eq!(TraceKind::of(&local), Some(TraceKind::Local));
|
||||
// One remote route makes the attempt remote
|
||||
let mixed = trace(vec![
|
||||
event(ATTEMPT, &[]),
|
||||
event(DOMAIN, &[(Key::Type, "local")]),
|
||||
event(HAM, &[]),
|
||||
event(DOMAIN, &[(Key::Type, "mx")]),
|
||||
]);
|
||||
assert_eq!(TraceKind::of(&mixed), Some(TraceKind::Remote));
|
||||
let relayed = trace(vec![
|
||||
event(ATTEMPT, &[]),
|
||||
event(DOMAIN, &[(Key::Type, "inbuxa-relay")]),
|
||||
]);
|
||||
assert_eq!(TraceKind::of(&relayed), Some(TraceKind::Remote));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn older_deliveries_by_events() {
|
||||
let local = trace(vec![
|
||||
event(ATTEMPT, &[]),
|
||||
event(DOMAIN, &[]),
|
||||
event(HAM, &[]),
|
||||
]);
|
||||
assert_eq!(TraceKind::of(&local), Some(TraceKind::Local));
|
||||
let remote = trace(vec![
|
||||
event(ATTEMPT, &[]),
|
||||
event(DOMAIN, &[]),
|
||||
event(CONNECT, &[]),
|
||||
]);
|
||||
assert_eq!(TraceKind::of(&remote), Some(TraceKind::Remote));
|
||||
// Failed before connecting: still a remote delivery
|
||||
let failed = trace(vec![event(ATTEMPT, &[]), event(DOMAIN, &[])]);
|
||||
assert_eq!(TraceKind::of(&failed), Some(TraceKind::Remote));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn names_round_trip() {
|
||||
for kind in [
|
||||
TraceKind::Received,
|
||||
TraceKind::Submitted,
|
||||
TraceKind::Remote,
|
||||
TraceKind::Local,
|
||||
] {
|
||||
assert_eq!(TraceKind::parse(kind.as_str()), Some(kind));
|
||||
}
|
||||
assert_eq!(TraceKind::parse("any"), None);
|
||||
assert_eq!(TraceKind::of(&trace(vec![])), None);
|
||||
}
|
||||
}
|
||||
@@ -224,7 +224,7 @@ pub(crate) async fn metric_query(
|
||||
|
||||
// ---- Traces (MON-10 to MON-17, MON-31, MON-32) ----
|
||||
|
||||
use common::telemetry::tracers::store::{MaybeTrace, decode_trace};
|
||||
use common::telemetry::tracers::store::{MaybeTrace, TraceKind, decode_trace};
|
||||
use registry::schema::structs::{Search, Trace, TraceValue};
|
||||
use store::{
|
||||
IterateParams,
|
||||
@@ -336,7 +336,7 @@ pub(crate) async fn trace_get(
|
||||
}
|
||||
let ids = match get.ids.take() {
|
||||
Some(ids) => ids,
|
||||
None => trace_ids(server, 0, u64::MAX, false, None)
|
||||
None => trace_ids(server, 0, u64::MAX, false, None, None)
|
||||
.await?
|
||||
.into_iter()
|
||||
.take(server.core.jmap.get_max_objects)
|
||||
@@ -353,13 +353,14 @@ pub(crate) async fn trace_get(
|
||||
}
|
||||
|
||||
/// Trace ids in a range, newest first unless `ascending`, optionally only
|
||||
/// those whose opening event is `event`.
|
||||
/// those whose opening event is `event` and whose kind is `kind`.
|
||||
async fn trace_ids(
|
||||
server: &common::Server,
|
||||
from_id: u64,
|
||||
to_id: u64,
|
||||
ascending: bool,
|
||||
event: Option<EventType>,
|
||||
kind: Option<TraceKind>,
|
||||
) -> trc::Result<Vec<u64>> {
|
||||
let from_id = from_id.max(trace_floor(server).await);
|
||||
let mut ids = Vec::new();
|
||||
@@ -387,6 +388,7 @@ async fn trace_ids(
|
||||
.next()
|
||||
.is_some_and(|first| first.event == event)
|
||||
})
|
||||
&& kind.is_none_or(|kind| TraceKind::of(&trace) == Some(kind))
|
||||
{
|
||||
ids.push(id);
|
||||
}
|
||||
@@ -397,7 +399,8 @@ async fn trace_ids(
|
||||
Ok(ids)
|
||||
}
|
||||
|
||||
/// `x:Trace/query`: `event` (the opening event), `text` and `queueId`
|
||||
/// `x:Trace/query`: `event` (the opening event), `kind` (received,
|
||||
/// submitted, remote or local; `any` for all, #397), `text` and `queueId`
|
||||
/// (through the search index, refused when trace search is off), and the
|
||||
/// timestamp comparisons.
|
||||
pub(crate) async fn trace_query(
|
||||
@@ -406,6 +409,7 @@ pub(crate) async fn trace_query(
|
||||
assert_server_level(req.access_token)?;
|
||||
let (mut from_id, mut to_id) = (0u64, u64::MAX);
|
||||
let mut event = None;
|
||||
let mut kind = None;
|
||||
let mut search: Vec<SearchFilter> = Vec::new();
|
||||
req.request
|
||||
.extract_filters(|property, op, value| match property {
|
||||
@@ -432,6 +436,17 @@ pub(crate) async fn trace_query(
|
||||
}
|
||||
None => false,
|
||||
},
|
||||
Property::Kind => match value.as_str() {
|
||||
Some("any") => true,
|
||||
Some(value) => match TraceKind::parse(value) {
|
||||
Some(parsed) => {
|
||||
kind = Some(parsed);
|
||||
true
|
||||
}
|
||||
None => false,
|
||||
},
|
||||
None => false,
|
||||
},
|
||||
Property::Text => match value.as_str() {
|
||||
Some(text) => {
|
||||
search.push(SearchFilter::has_text(
|
||||
@@ -477,7 +492,15 @@ pub(crate) async fn trace_query(
|
||||
.details("Traces sort by timestamp only"));
|
||||
}
|
||||
|
||||
let mut ids = trace_ids(req.server, from_id, to_id, params.sort_ascending, event).await?;
|
||||
let mut ids = trace_ids(
|
||||
req.server,
|
||||
from_id,
|
||||
to_id,
|
||||
params.sort_ascending,
|
||||
event,
|
||||
kind,
|
||||
)
|
||||
.await?;
|
||||
if !search.is_empty() {
|
||||
let settings = req
|
||||
.server
|
||||
|
||||
@@ -991,6 +991,8 @@ pub enum Property {
|
||||
IntrospectionClientSecret = 938,
|
||||
// inbuxa: relay spec, RL-24
|
||||
Relays = 939,
|
||||
// inbuxa: #397: a trace's kind, as x:Trace/query filters it
|
||||
Kind = 940,
|
||||
RequireClientRegistration = 615,
|
||||
RequireScopes = 608,
|
||||
RequireTls = 525,
|
||||
|
||||
@@ -1142,6 +1142,7 @@ impl EnumImpl for Property {
|
||||
b"introspectionClientId" => Property::IntrospectionClientId,
|
||||
b"introspectionClientSecret" => Property::IntrospectionClientSecret,
|
||||
b"relays" => Property::Relays,
|
||||
b"kind" => Property::Kind,
|
||||
b"requireClientRegistration" => Property::RequireClientRegistration,
|
||||
b"requireScopes" => Property::RequireScopes,
|
||||
b"requireTls" => Property::RequireTls,
|
||||
@@ -2087,6 +2088,7 @@ impl EnumImpl for Property {
|
||||
Property::IntrospectionClientId => "introspectionClientId",
|
||||
Property::IntrospectionClientSecret => "introspectionClientSecret",
|
||||
Property::Relays => "relays",
|
||||
Property::Kind => "kind",
|
||||
Property::RequireClientRegistration => "requireClientRegistration",
|
||||
Property::RequireScopes => "requireScopes",
|
||||
Property::RequireTls => "requireTls",
|
||||
@@ -3036,6 +3038,7 @@ impl EnumImpl for Property {
|
||||
937 => Some(Property::IntrospectionClientId),
|
||||
938 => Some(Property::IntrospectionClientSecret),
|
||||
939 => Some(Property::Relays),
|
||||
940 => Some(Property::Kind),
|
||||
615 => Some(Property::RequireClientRegistration),
|
||||
608 => Some(Property::RequireScopes),
|
||||
525 => Some(Property::RequireTls),
|
||||
|
||||
@@ -305,6 +305,14 @@ impl QueuedMessage {
|
||||
Delivery(DeliveryEvent::DomainDeliveryStart),
|
||||
SpanId = message.span_id,
|
||||
Domain = domain.to_string(),
|
||||
// inbuxa: #397: delivery history tells local deliveries from
|
||||
// remote ones by this
|
||||
Type = match route {
|
||||
RoutingStrategy::Local => "local",
|
||||
RoutingStrategy::Mx(_) => "mx",
|
||||
RoutingStrategy::Relay(_) => "relay",
|
||||
RoutingStrategy::InbuxaRelay(_) => "inbuxa-relay",
|
||||
},
|
||||
);
|
||||
|
||||
// Build envelope
|
||||
|
||||
@@ -22,6 +22,7 @@ use crate::{
|
||||
};
|
||||
use common::{Server, config::mailstore::spamfilter::SpamFilterAction};
|
||||
use std::{fmt::Write, future::Future, vec};
|
||||
use trc::SpamEvent;
|
||||
|
||||
pub trait SpamFilterAnalyzeScore: Sync + Send {
|
||||
fn spam_filter_finalize(
|
||||
@@ -74,9 +75,21 @@ impl SpamFilterAnalyzeScore for Server {
|
||||
let score = match self.core.spam.lists.scores.get(tag) {
|
||||
Some(SpamFilterAction::Allow(score)) => *score,
|
||||
Some(SpamFilterAction::Discard) => {
|
||||
spam_result_event(
|
||||
ctx.input.span_id,
|
||||
"discard",
|
||||
ctx.result.score,
|
||||
vec![tag.to_string()],
|
||||
);
|
||||
return SpamFilterAction::Discard;
|
||||
}
|
||||
Some(SpamFilterAction::Reject) => {
|
||||
spam_result_event(
|
||||
ctx.input.span_id,
|
||||
"reject",
|
||||
ctx.result.score,
|
||||
vec![tag.to_string()],
|
||||
);
|
||||
return SpamFilterAction::Reject;
|
||||
}
|
||||
None | Some(SpamFilterAction::Disabled) => 0.0,
|
||||
@@ -135,17 +148,33 @@ impl SpamFilterAnalyzeScore for Server {
|
||||
}
|
||||
}
|
||||
|
||||
results.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap().then_with(|| a.0.cmp(b.0)));
|
||||
if self.core.spam.scores.reject_threshold > 0.0
|
||||
&& final_score >= self.core.spam.scores.reject_threshold
|
||||
{
|
||||
spam_result_event(
|
||||
ctx.input.span_id,
|
||||
"reject",
|
||||
final_score,
|
||||
rule_list(&results),
|
||||
);
|
||||
SpamFilterAction::Reject
|
||||
} else if self.core.spam.scores.discard_threshold > 0.0
|
||||
&& final_score >= self.core.spam.scores.discard_threshold
|
||||
{
|
||||
spam_result_event(
|
||||
ctx.input.span_id,
|
||||
"discard",
|
||||
final_score,
|
||||
rule_list(&results),
|
||||
);
|
||||
SpamFilterAction::Discard
|
||||
} else {
|
||||
let is_spam = final_score >= self.core.spam.scores.spam_threshold;
|
||||
let class = if is_spam { "spam" } else { "ham" };
|
||||
spam_result_event(ctx.input.span_id, class, final_score, rule_list(&results));
|
||||
|
||||
let mut headers = String::with_capacity(header_len + 40);
|
||||
results.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap().then_with(|| a.0.cmp(b.0)));
|
||||
headers.push_str("X-Spam-Result: ");
|
||||
for (idx, (tag, score)) in results.into_iter().enumerate() {
|
||||
if idx > 0 {
|
||||
@@ -163,9 +192,6 @@ impl SpamFilterAnalyzeScore for Server {
|
||||
));
|
||||
}
|
||||
|
||||
let is_spam = final_score >= self.core.spam.scores.spam_threshold;
|
||||
let class = if is_spam { "spam" } else { "ham" };
|
||||
|
||||
if avg_confidence != 0.0 {
|
||||
let _ = write!(
|
||||
&mut headers,
|
||||
@@ -270,6 +296,56 @@ impl SpamFilterAnalyzeScore for Server {
|
||||
}
|
||||
}
|
||||
|
||||
/// inbuxa: #397: the verdict, the score and the rules the message hit, at
|
||||
/// info level, so a stored trace shows why a message scored the way it did.
|
||||
/// `rules` is the X-Spam-Result list (highest points first), or, when one rule rejects or discards
|
||||
/// on its own, that rule's name.
|
||||
fn spam_result_event(span_id: u64, verdict: &'static str, score: f32, rules: Vec<String>) {
|
||||
trc::event!(
|
||||
Spam(SpamEvent::Result),
|
||||
SpanId = span_id,
|
||||
Result = verdict,
|
||||
Value = score,
|
||||
Details = rules,
|
||||
);
|
||||
}
|
||||
|
||||
/// Each rule as X-Spam-Result writes it, `TAG (points)`, but highest points
|
||||
/// first, so the rules that moved the score lead (the header lists lowest
|
||||
/// first).
|
||||
fn rule_list(results: &[(&str, f32)]) -> Vec<String> {
|
||||
let mut rules = results.to_vec();
|
||||
rules.sort_by(|a, b| b.1.total_cmp(&a.1).then_with(|| a.0.cmp(b.0)));
|
||||
rules
|
||||
.into_iter()
|
||||
.map(|(tag, score)| format!("{tag} ({score:.2})"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn rule_list_puts_the_highest_points_first() {
|
||||
let results = [
|
||||
("B_ZERO", 0.0),
|
||||
("RCVD", 0.1),
|
||||
("A_ZERO", 0.0),
|
||||
("HAM", -1.5),
|
||||
("RBL", 3.0),
|
||||
];
|
||||
assert_eq!(
|
||||
super::rule_list(&results),
|
||||
[
|
||||
"RBL (3.00)",
|
||||
"RCVD (0.10)",
|
||||
"A_ZERO (0.00)",
|
||||
"B_ZERO (0.00)",
|
||||
"HAM (-1.50)"
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub trait ConfidenceStore {
|
||||
fn spam_tag(&self) -> &'static str;
|
||||
}
|
||||
|
||||
@@ -14,10 +14,11 @@
|
||||
// coordinator's connection events; 647 and 648 are the audit log's
|
||||
// (audit-hold-lock spec, AU-3, AU-8); 649 is cluster.node-id-reassigned;
|
||||
// 650 to 659 are inbuxa Relay's (relay spec, RL-23, RL-T23); 660 is
|
||||
// store.blob-read-through; 661 is smtp.directory-no-answer (#364).
|
||||
// store.blob-read-through; 661 is smtp.directory-no-answer (#364); 662 is
|
||||
// spam.result (#397).
|
||||
// TOTAL_EVENT_COUNT sizes the collector's per-event tables, so it is one
|
||||
// more than the highest id; the test below holds it to the variant list
|
||||
pub const TOTAL_EVENT_COUNT: usize = 662;
|
||||
pub const TOTAL_EVENT_COUNT: usize = 663;
|
||||
pub const TOTAL_METRIC_COUNT: usize = 369;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
@@ -828,6 +829,9 @@ pub enum SpamEvent {
|
||||
TrainSampleAdded = 143,
|
||||
TrainSampleNotFound = 491,
|
||||
Classify = 490,
|
||||
/// inbuxa: #397: the spam filter's verdict, score and the rules the
|
||||
/// message hit.
|
||||
Result = 662,
|
||||
ModelLoaded = 589,
|
||||
ModelNotReady = 496,
|
||||
ModelNotFound = 497,
|
||||
|
||||
@@ -582,6 +582,8 @@ impl EventType {
|
||||
b"spam.train-sample-added" => EventType::Spam(SpamEvent::TrainSampleAdded),
|
||||
b"spam.train-sample-not-found" => EventType::Spam(SpamEvent::TrainSampleNotFound),
|
||||
b"spam.classify" => EventType::Spam(SpamEvent::Classify),
|
||||
// inbuxa: #397
|
||||
b"spam.result" => EventType::Spam(SpamEvent::Result),
|
||||
b"spam.model-loaded" => EventType::Spam(SpamEvent::ModelLoaded),
|
||||
b"spam.model-not-ready" => EventType::Spam(SpamEvent::ModelNotReady),
|
||||
b"spam.model-not-found" => EventType::Spam(SpamEvent::ModelNotFound),
|
||||
@@ -1381,6 +1383,8 @@ impl EventType {
|
||||
EventType::Spam(SpamEvent::TrainSampleAdded) => "spam.train-sample-added",
|
||||
EventType::Spam(SpamEvent::TrainSampleNotFound) => "spam.train-sample-not-found",
|
||||
EventType::Spam(SpamEvent::Classify) => "spam.classify",
|
||||
// inbuxa: #397
|
||||
EventType::Spam(SpamEvent::Result) => "spam.result",
|
||||
EventType::Spam(SpamEvent::ModelLoaded) => "spam.model-loaded",
|
||||
EventType::Spam(SpamEvent::ModelNotReady) => "spam.model-not-ready",
|
||||
EventType::Spam(SpamEvent::ModelNotFound) => "spam.model-not-found",
|
||||
@@ -2073,6 +2077,8 @@ impl EventType {
|
||||
EventType::Spam(SpamEvent::TrainSampleAdded) => 143,
|
||||
EventType::Spam(SpamEvent::TrainSampleNotFound) => 491,
|
||||
EventType::Spam(SpamEvent::Classify) => 490,
|
||||
// inbuxa: #397
|
||||
EventType::Spam(SpamEvent::Result) => 662,
|
||||
EventType::Spam(SpamEvent::ModelLoaded) => 589,
|
||||
EventType::Spam(SpamEvent::ModelNotReady) => 496,
|
||||
EventType::Spam(SpamEvent::ModelNotFound) => 497,
|
||||
@@ -2785,6 +2791,8 @@ impl EventType {
|
||||
143 => Some(EventType::Spam(SpamEvent::TrainSampleAdded)),
|
||||
491 => Some(EventType::Spam(SpamEvent::TrainSampleNotFound)),
|
||||
490 => Some(EventType::Spam(SpamEvent::Classify)),
|
||||
// inbuxa: #397
|
||||
662 => Some(EventType::Spam(SpamEvent::Result)),
|
||||
589 => Some(EventType::Spam(SpamEvent::ModelLoaded)),
|
||||
496 => Some(EventType::Spam(SpamEvent::ModelNotReady)),
|
||||
497 => Some(EventType::Spam(SpamEvent::ModelNotFound)),
|
||||
@@ -3151,6 +3159,8 @@ impl EventType {
|
||||
EventType::Spam(SpamEvent::ModelNotReady) => Level::Info,
|
||||
EventType::Spam(SpamEvent::ModelNotFound) => Level::Info,
|
||||
EventType::Spam(SpamEvent::RulesUpdated) => Level::Info,
|
||||
// inbuxa: #397
|
||||
EventType::Spam(SpamEvent::Result) => Level::Info,
|
||||
EventType::Store(StoreEvent::BlobStorePurged) => Level::Info,
|
||||
EventType::Store(StoreEvent::DataStorePurged) => Level::Info,
|
||||
EventType::TaskManager(TaskManagerEvent::TaskQueued) => Level::Info,
|
||||
@@ -3984,6 +3994,8 @@ impl EventType {
|
||||
EventType::Spam(SpamEvent::TrainSampleAdded) => "New training sample added",
|
||||
EventType::Spam(SpamEvent::TrainSampleNotFound) => "Training sample not found",
|
||||
EventType::Spam(SpamEvent::Classify) => "Classifying message for spam",
|
||||
// inbuxa: #397
|
||||
EventType::Spam(SpamEvent::Result) => "Spam filter result",
|
||||
EventType::Spam(SpamEvent::ModelLoaded) => "Spam classifier model loaded",
|
||||
EventType::Spam(SpamEvent::ModelNotReady) => "Spam classifier model not ready",
|
||||
EventType::Spam(SpamEvent::ModelNotFound) => "Spam classifier model not found",
|
||||
@@ -4974,6 +4986,8 @@ impl EventType {
|
||||
EventType::Spam(SpamEvent::TrainSampleAdded),
|
||||
EventType::Spam(SpamEvent::TrainSampleNotFound),
|
||||
EventType::Spam(SpamEvent::Classify),
|
||||
// inbuxa: #397
|
||||
EventType::Spam(SpamEvent::Result),
|
||||
EventType::Spam(SpamEvent::ModelLoaded),
|
||||
EventType::Spam(SpamEvent::ModelNotReady),
|
||||
EventType::Spam(SpamEvent::ModelNotFound),
|
||||
|
||||
@@ -224,6 +224,22 @@ unchanged.
|
||||
So searching `example.org` finds every trace to or from that domain, as the
|
||||
upstream suite expects. With `indexTelemetry` off nothing is indexed, and
|
||||
the `text` and `queueId` filters are refused (see "Interfaces").
|
||||
- **MON-40.** (#397) A trace has a kind, worked out from its events when it
|
||||
is read, so traces stored earlier have one too. An SMTP session is
|
||||
`submitted` when it holds `auth.success` and `received` otherwise. A
|
||||
delivery attempt is `local` when every `delivery.domain-delivery-start` in
|
||||
it names the `local` route in `Type` (`local`, `mx`, `relay` or
|
||||
`inbuxa-relay`), and `remote` when any names another. A trace from before
|
||||
those types is `local` when it holds a `message-ingest.*` event and no
|
||||
`delivery.connect`, and `remote` otherwise. The inbound list shows
|
||||
`received` and the outbound list `remote` by default. A filter shows the
|
||||
other kind, or both.
|
||||
- **MON-41.** (#397) The spam filter raises `spam.result` (info level) for
|
||||
every message it scores. `Result` holds the verdict (`ham`, `spam`, `reject`
|
||||
or `discard`), `Value` the score, and `Details` each rule the message hit
|
||||
with its points, highest first (X-Spam-Result lists them lowest first). When one rule rejects or
|
||||
discards a message by itself, `Details` names only that rule. An inbound
|
||||
session's trace therefore shows why its message scored the way it did.
|
||||
|
||||
### Retention and growth
|
||||
|
||||
@@ -385,7 +401,9 @@ unchanged.
|
||||
- `x:Trace/get` and `/query`. Filters: `text`, `timestamp` (comparison names
|
||||
such as `timestampIsGreaterThan`), `queueId`, `event` (the type of the
|
||||
trace's opening event: the list views use `smtp.connection-start` and
|
||||
`delivery.attempt-start`). Sort by `timestamp`, newest first by default.
|
||||
`delivery.attempt-start`), `kind` (`received`, `submitted`, `remote`,
|
||||
`local`, or `any` for all; MON-40). Sort by `timestamp`, newest first by
|
||||
default.
|
||||
`/set` refuses create and update, and allows destroy (MON-32).
|
||||
- `x:Metric/get` and `/query`. Filters: `metric` (one name or a list) and
|
||||
the `timestamp` comparisons `timestampIsGreaterThan`,
|
||||
@@ -549,7 +567,7 @@ they cover tests 10 and 12 (`tracing.rs`), 13 (`metrics.rs`) and 18
|
||||
(`alerts.rs`). Tests 1, 3, 5 to 9, 11, 14 to 17, 19 to 21, 23 and 24 pass as
|
||||
`tests/src/system/monitoring.rs`.
|
||||
|
||||
- **MON-1 to MON-39:** built, with the limits below.
|
||||
- **MON-1 to MON-41:** built, with the limits below.
|
||||
- **Test 26 (compat)** is written as `monitoring_compat`, ignored, and unrun
|
||||
until a copy of INBUXA's data is provided. It checks observed 1, reads the
|
||||
old history without an error, and purges it. It deletes history, so only
|
||||
|
||||
Binary file not shown.
@@ -1 +1 @@
|
||||
d5R3iz87YIYY8m_r62Kd4-dlqkkVUHt1lz7rKB9fBO0
|
||||
UDW2s5eJMuFvg1FJkP9HP4a1IApTtrR0Vj9awc3VjhY
|
||||
@@ -168,6 +168,37 @@ pub async fn test(test: &mut TestServer) {
|
||||
"test 9: to"
|
||||
);
|
||||
|
||||
// #397: the session came from outside and the attempt only filed the
|
||||
// message here, so each list's default leaves out the other kind
|
||||
for (event, kind, expected) in [
|
||||
("smtp.connection-start", "received", 1),
|
||||
("smtp.connection-start", "submitted", 0),
|
||||
("smtp.connection-start", "any", 1),
|
||||
("delivery.attempt-start", "local", 1),
|
||||
("delivery.attempt-start", "remote", 0),
|
||||
("delivery.attempt-start", "any", 1),
|
||||
] {
|
||||
let response = admin
|
||||
.jmap_method_call(
|
||||
"x:Trace/query",
|
||||
json!({"filter": {"event": event, "kind": kind}}),
|
||||
)
|
||||
.await;
|
||||
let found = response
|
||||
.0
|
||||
.pointer("/methodResponses/0/1/ids")
|
||||
.and_then(|ids| ids.as_array())
|
||||
.map(|ids| ids.len());
|
||||
assert_eq!(found, Some(expected), "#397: {event} {kind}: {response:?}");
|
||||
}
|
||||
let response = admin
|
||||
.jmap_method_call("x:Trace/query", json!({"filter": {"kind": "sideways"}}))
|
||||
.await;
|
||||
assert!(
|
||||
response.0.pointer("/methodResponses/0/1/ids").is_none(),
|
||||
"#397: an unknown kind is refused: {response:?}"
|
||||
);
|
||||
|
||||
// MON-16: the queueId filter finds the traces that name a queue id (the
|
||||
// session that queued the message and its delivery attempt) through the
|
||||
// search index, given as a string or a number (the index column is an
|
||||
|
||||
Reference in new issue
Block a user