Monitoring: trace history, trace search, and x:Trace over the stored traces (MON-10 to MON-17, MON-32)

A lossy collector subscriber keeps each inbound SMTP session that reached
MAIL FROM and each delivery attempt, info level and above and never raw
I/O, at most 1,000 events with strings cut at 4 KiB, and writes it when
the span closes as an x:Trace under the telemetry key class, scheduling
its indexing. The index task builds a document of event types, queue ids
and keywords when indexTelemetry is on. x:Trace/get derives timestamp,
from, to and size; /query filters by opening event, text, queueId and
time; /set destroys only. The data purge honours holdTracesFor. The shared
tracing and webhook suites run.

crates/common/src/config/telemetry.rs

@@ -40,6 +40,15 @@ pub enum TelemetrySubscriberType {
     Webhook(WebhookTracer),
     #[cfg(unix)]
     JournalTracer(crate::telemetry::tracers::journald::Subscriber),
+    // inbuxa: MON-10: trace history
+    StoreTracer(StoreTracer),
+}
+
+/// Where trace history goes: traces to `tracing`, index tasks to `data`.
+#[derive(Debug)]
+pub struct StoreTracer {
+    pub tracing: store::Store,
+    pub data: store::Store,
 }
 
 #[derive(Debug)]
@@ -141,8 +150,7 @@ impl Telemetry {
 }
 
 impl Tracers {
-    // inbuxa: `_storage` is unused until monitoring history (stored traces and metrics) is rebuilt
-    pub async fn parse(bp: &mut Bootstrap, _storage: &Storage) -> Self {
+    pub async fn parse(bp: &mut Bootstrap, storage: &Storage) -> Self {
         let mut custom_levels = AHashMap::new();
         let mut tracers: Vec<TelemetrySubscriber> = Vec::new();
         let mut global_interests = Interests::default();
@@ -387,6 +395,7 @@ impl Tracers {
                     TelemetrySubscriberType::JournalTracer(_) => {
                         EventType::Telemetry(TelemetryEvent::JournalError).into()
                     }
+                    TelemetrySubscriberType::StoreTracer(_) => None,
                 };
 
                 // Parse disabled events
@@ -478,6 +487,36 @@ impl Tracers {
                 }
             }
 
+            // inbuxa: MON-10 to MON-12: trace history, when a tracing store is set:
+            // info and above, the span edges and MAIL FROM, never raw I/O
+            if !storage.tracing.is_none() {
+                let mut interests = Interests::default();
+                for event_type in EventType::variants() {
+                    let event_level = custom_levels
+                        .get(event_type)
+                        .copied()
+                        .unwrap_or(event_type.level());
+                    if !event_type.is_raw_io()
+                        && (Level::Info.is_contained(event_level)
+                            || event_type.is_span_start()
+                            || event_type.is_span_end()
+                            || event_type.as_str().starts_with("smtp.mail-from"))
+                    {
+                        interests.set(event_type.to_id() as usize);
+                        global_interests.set(event_type.to_id() as usize);
+                    }
+                }
+                tracers.push(TelemetrySubscriber {
+                    id: "trace-history".to_string(),
+                    interests,
+                    typ: TelemetrySubscriberType::StoreTracer(StoreTracer {
+                        tracing: storage.tracing.clone(),
+                        data: storage.data.clone(),
+                    }),
+                    lossy: true,
+                });
+            }
+
             #[cfg(feature = "dev_mode")]
             if let Ok(level) = std::env::var("LOG") {
                 let level = Level::from_str(&level).expect("Invalid LOG level");