diff --git a/crates/common/src/config/telemetry.rs b/crates/common/src/config/telemetry.rs index 28865310..5fd8351c 100644 --- a/crates/common/src/config/telemetry.rs +++ b/crates/common/src/config/telemetry.rs @@ -17,7 +17,7 @@ use opentelemetry_otlp::{ use opentelemetry_sdk::{Resource, metrics::Temporality}; use opentelemetry_semantic_conventions::resource::SERVICE_VERSION; use registry::schema::{ - enums::{EventPolicy, LogRotateFrequency}, + enums::{EventPolicy, LogFormat, LogRotateFrequency}, prelude::ObjectType, structs::{self, EventTracingLevel, MetricsPrometheus, Tracer, WebHook}, }; @@ -87,6 +87,8 @@ pub struct LogTracer { pub rotate: RotationStrategy, pub ansi: bool, pub multiline: bool, + // inbuxa: #456, one JSON object per line instead of text + pub json: bool, } #[derive(Debug)] @@ -196,6 +198,7 @@ impl Tracers { }, ansi: tracer.ansi, multiline: tracer.multiline, + json: tracer.format == LogFormat::Json, }) } Tracer::Stdout(tracer) if tracer.enable => { diff --git a/crates/common/src/telemetry/tracers/log.rs b/crates/common/src/telemetry/tracers/log.rs index 930fdbe2..17838bb4 100644 --- a/crates/common/src/telemetry/tracers/log.rs +++ b/crates/common/src/telemetry/tracers/log.rs @@ -13,9 +13,66 @@ use crate::config::telemetry::{LogTracer, RotationStrategy}; use mail_parser::DateTime; use tokio::{ fs::{File, OpenOptions}, - io::BufWriter, + io::{AsyncWriteExt, BufWriter}, }; -use trc::{TelemetryEvent, ipc::subscriber::SubscriberBuilder, serializers::text::FmtWriter}; +use trc::{ + Event, EventDetails, TelemetryEvent, + ipc::subscriber::SubscriberBuilder, + serializers::{json::JsonLogLine, text::FmtWriter}, +}; + +// inbuxa (#456): a Log tracer writes text, as upstream, or one JSON object +// per line. +enum LogWriter { + Text(FmtWriter>), + Json(BufWriter, Vec), +} + +impl LogWriter { + fn new(writer: BufWriter, settings: &LogTracer) -> Self { + if settings.json { + LogWriter::Json(writer, Vec::with_capacity(512)) + } else { + LogWriter::Text( + FmtWriter::new(writer) + .with_ansi(settings.ansi) + .with_multiline(settings.multiline), + ) + } + } + + async fn write(&mut self, event: &Event) -> std::io::Result<()> { + match self { + LogWriter::Text(writer) => writer.write(event).await, + LogWriter::Json(writer, line) => { + line.clear(); + json_log_line(event, line)?; + writer.write_all(line).await + } + } + } + + async fn flush(&mut self) -> std::io::Result<()> { + match self { + LogWriter::Text(writer) => writer.flush().await, + LogWriter::Json(writer, _) => writer.flush().await, + } + } + + fn update_writer(&mut self, new: BufWriter) { + match self { + LogWriter::Text(writer) => writer.update_writer(new), + LogWriter::Json(writer, _) => *writer = new, + } + } +} + +/// inbuxa (#456): an event as one line of JSON, newline included. +pub(crate) fn json_log_line(event: &Event, out: &mut Vec) -> std::io::Result<()> { + serde_json::to_writer(&mut *out, &JsonLogLine(event)).map_err(std::io::Error::other)?; + out.push(b'\n'); + Ok(()) +} // inbuxa: when a Log tracer is started over on the same files (its rotation // or format changed), the new one waits for the old one to write what it @@ -39,9 +96,7 @@ pub(crate) fn spawn_log_tracer(builder: SubscriberBuilder, settings: LogTracer) } if let Some(writer) = settings.build_writer().await { - let mut buf = FmtWriter::new(writer) - .with_ansi(settings.ansi) - .with_multiline(settings.multiline); + let mut buf = LogWriter::new(writer, &settings); let mut roatation_timestamp = settings.next_rotation(); while let Some(events) = rx.recv().await { @@ -159,3 +214,49 @@ impl LogTracer { } } } + +#[cfg(test)] +mod tests { + use super::json_log_line; + use trc::{Event, EventDetails, EventType, Key, Level, SmtpEvent, Value}; + + // inbuxa (#456): a JSON line is one object with typed fields + #[test] + fn json_line_is_one_typed_object() { + let event = Event { + inner: EventDetails { + typ: EventType::Smtp(SmtpEvent::ConnectionStart), + timestamp: 1_794_996_000, + level: Level::Info, + span: None, + }, + keys: vec![ + (Key::RemoteIp, Value::String("192.0.2.1".into())), + (Key::RemotePort, Value::UInt(25)), + ( + Key::Details, + Value::String("a \"quoted\" word\nand a line".into()), + ), + ( + Key::Contents, + Value::Array(vec![Value::Bool(true), Value::Int(-1)]), + ), + ], + }; + let mut out = Vec::new(); + json_log_line(&event, &mut out).unwrap(); + + let text = String::from_utf8(out).unwrap(); + assert!(text.ends_with('\n')); + assert_eq!(text.matches('\n').count(), 1, "one line: {text}"); + let json: serde_json::Value = serde_json::from_str(&text).unwrap(); + assert_eq!(json["timestamp"], "2026-11-18T10:00:00Z"); + assert_eq!(json["level"], "info"); + assert_eq!(json["event"], "smtp.connection-start"); + assert!(json["description"].as_str().is_some_and(|d| !d.is_empty())); + assert_eq!(json["data"]["remoteIp"], "192.0.2.1"); + assert_eq!(json["data"]["remotePort"], 25); + assert_eq!(json["data"]["details"], "a \"quoted\" word\nand a line"); + assert_eq!(json["data"]["contents"], serde_json::json!([true, -1])); + } +} diff --git a/crates/jmap/src/registry/mapping/log.rs b/crates/jmap/src/registry/mapping/log.rs index f08f27b2..f04dcbfb 100644 --- a/crates/jmap/src/registry/mapping/log.rs +++ b/crates/jmap/src/registry/mapping/log.rs @@ -252,6 +252,17 @@ struct LogHeader<'x> { } fn log_header(line: &str) -> Option> { + // inbuxa (#456): a Log tracer writing JSON lines + if line.starts_with('{') { + let entry = serde_json::from_str::>(line).ok()?; + return Some(LogHeader { + timestamp: DateTime::parse_from_rfc3339(entry.timestamp) + .ok()? + .timestamp(), + level: TracingLevel::parse(&entry.level.to_ascii_lowercase()).map(|l| l as u8), + event: entry.event, + }); + } if !line.as_bytes().first()?.is_ascii_digit() { return None; } @@ -409,7 +420,53 @@ pub(crate) fn read_log_entries( Ok(entries) } +/// inbuxa (#456): the fields of a JSON log line that a query filters on. +#[derive(serde::Deserialize)] +struct JsonLogHead<'x> { + #[serde(borrow)] + timestamp: &'x str, + #[serde(borrow)] + level: &'x str, + #[serde(borrow)] + event: &'x str, +} + +/// inbuxa (#456): a JSON log line, its `data` handed back as a JSON object +/// in `details`, so a reader gets the fields exactly as they were written. +#[derive(serde::Deserialize)] +struct JsonLogEntry<'x> { + #[serde(borrow)] + timestamp: &'x str, + #[serde(borrow)] + level: &'x str, + #[serde(borrow)] + event: &'x str, + #[serde(default)] + data: serde_json::Map, +} + +fn log_from_json_line(line: &str) -> Option { + let entry = serde_json::from_str::>(line.trim_end()).ok()?; + Some(Log { + timestamp: UTCDateTime::from_timestamp( + DateTime::parse_from_rfc3339(entry.timestamp) + .ok()? + .timestamp(), + ), + level: TracingLevel::parse(&entry.level.to_ascii_lowercase()).unwrap_or(TracingLevel::Info), + event: EventType::parse(entry.event)?, + details: if entry.data.is_empty() { + String::new() + } else { + serde_json::Value::Object(entry.data).to_string() + }, + }) +} + fn log_from_line(line: &str) -> Option { + if line.starts_with('{') { + return log_from_json_line(line); + } let line = strip_ansi(line); let (timestamp, rest) = line.split_once(' ')?; let timestamp = DateTime::parse_from_rfc3339(timestamp).ok()?; @@ -636,7 +693,7 @@ impl Iterator for RevLines { #[cfg(test)] mod tests { - use super::{LogFilter, read_log_offsets}; + use super::{LogFilter, read_log_entries, read_log_offsets}; #[test] fn limit_zero_reads_no_entries() { @@ -702,4 +759,133 @@ mod tests { assert_eq!(window, 2); assert_eq!(after, 2); } + + // inbuxa (#456): files written as JSON lines are queried and read like + // text ones, and a mix of both (a tracer switched over) reads in order + #[test] + fn reads_json_lines() { + let dir = std::env::temp_dir().join(format!("inbuxa-log-json-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write( + dir.join("inbuxa.log.2026-11-18"), + concat!( + "2026-11-18T10:00:00Z INFO SMTP connection started (smtp.connection-start) remoteIp = 192.0.2.1\n", + r#"{"timestamp":"2026-11-18T10:00:01Z","level":"warn","event":"store.data-corruption","description":"Data corruption","data":{"details":"a \"quoted\" word, with a comma"}}"#, + "\n", + r#"{"timestamp":"2026-11-18T10:00:02Z","level":"error","event":"imap.connection-end","description":"x","data":{}}"#, + "\n", + r#"{"timestamp":"2026-11-18T10:00:03Z","level":"debug","event":"smtp.rcpt-to","description":"y","data":{"to":["a@example.org"],"size":12}}"#, + "\n", + ), + ) + .unwrap(); + let ids = |filter: LogFilter| read_log_offsets(&dir, &filter, 0, 10).unwrap(); + + let all = ids(LogFilter::default()); + let at_least_warn = ids(LogFilter { + level: Some(1), + ..Default::default() + }); + let smtp = ids(LogFilter { + event: Some("smtp".into()), + ..Default::default() + }); + let window = ids(LogFilter { + from: 1_794_996_001, + to: 1_794_996_002, + ..Default::default() + }); + let text = ids(LogFilter { + text: Some("comma".into()), + ..Default::default() + }); + let entries = read_log_entries(&dir, Some(all.clone()), 10).unwrap(); + std::fs::remove_dir_all(&dir).unwrap(); + + assert_eq!(all.len(), 4); + assert_eq!(at_least_warn.len(), 2, "warn and error"); + assert_eq!(smtp.len(), 2, "smtp.* only"); + assert_eq!(window.len(), 2); + assert_eq!(text.len(), 1); + + // Newest first + let events = entries + .iter() + .map(|(_, log)| log.event.as_str()) + .collect::>(); + assert_eq!( + events, + [ + "smtp.rcpt-to", + "imap.connection-end", + "store.data-corruption", + "smtp.connection-start" + ] + ); + let details = |i: usize| entries[i].1.details.as_str(); + let rcpt: serde_json::Value = serde_json::from_str(details(0)).unwrap(); + assert_eq!( + rcpt, + serde_json::json!({"to": ["a@example.org"], "size": 12}) + ); + assert_eq!(details(1), "", "no fields, no details"); + let corruption: serde_json::Value = serde_json::from_str(details(2)).unwrap(); + assert_eq!(corruption["details"], "a \"quoted\" word, with a comma"); + assert_eq!(details(3), "remoteIp = 192.0.2.1"); + assert_eq!(entries[2].1.level as u8, 1); + } + + // inbuxa (#456): what the Log tracer writes as JSON reads back exactly + #[test] + fn json_tracer_output_reads_back() { + use trc::{Event, EventDetails, EventType, Key, Level, SmtpEvent, Value}; + + let event = Event { + inner: EventDetails { + typ: EventType::Smtp(SmtpEvent::RcptTo), + timestamp: 1_794_996_000, + level: Level::Debug, + span: Some(std::sync::Arc::new(Event { + inner: EventDetails { + typ: EventType::Smtp(SmtpEvent::ConnectionStart), + timestamp: 1_794_995_999, + level: Level::Info, + span: None, + }, + keys: vec![(Key::RemoteIp, Value::String("192.0.2.1".into()))], + })), + }, + keys: vec![(Key::To, Value::String("a \"b\", c@example.org".into()))], + }; + let mut line = serde_json::to_string(&trc::serializers::json::JsonLogLine(&event)).unwrap(); + line.push('\n'); + + let dir = std::env::temp_dir().join(format!("inbuxa-log-rt-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("inbuxa.log"), &line).unwrap(); + let ids = read_log_offsets( + &dir, + &LogFilter { + level: Some(3), + event: Some("smtp.rcpt-to".into()), + ..Default::default() + }, + 0, + 10, + ) + .unwrap(); + let entries = read_log_entries(&dir, Some(ids), 10).unwrap(); + std::fs::remove_dir_all(&dir).unwrap(); + + assert_eq!(entries.len(), 1); + let log = &entries[0].1; + assert_eq!(log.event, EventType::Smtp(SmtpEvent::RcptTo)); + assert_eq!(log.level as u8, 3); + assert_eq!(log.timestamp.timestamp(), 1_794_996_000); + let data: serde_json::Value = serde_json::from_str(&log.details).unwrap(); + assert_eq!( + data, + serde_json::json!({"to": "a \"b\", c@example.org", "remoteIp": "192.0.2.1"}) + ); + } } diff --git a/crates/registry/src/inbuxa_tests.rs b/crates/registry/src/inbuxa_tests.rs index c12ddf98..5a4f2a91 100644 --- a/crates/registry/src/inbuxa_tests.rs +++ b/crates/registry/src/inbuxa_tests.rs @@ -9,11 +9,11 @@ use crate::{ pickle::{Pickle, PickledStream}, schema::{ - enums::{MtaRouteType, OidcOpaqueTokens}, + enums::{LogFormat, MtaRouteType, OidcOpaqueTokens}, prelude::{ObjectImpl, ObjectInner, ObjectType}, structs::{ Bootstrap, Directory, DirectoryBootstrap, Domain, Enterprise, MtaRoute, MtaRouteCommon, - MtaRouteInbuxaRelay, OidcDirectory, Tenant, + MtaRouteInbuxaRelay, OidcDirectory, Tenant, Tracer, TracerLog, }, }, types::{EnumImpl, map::Map}, @@ -32,6 +32,26 @@ fn pickle_without_opaque_tokens(directory: &OidcDirectory, out: &mut Vec) { directory.member_tenant_id.pickle(out); } +/// A tracer as it was stored before a Log tracer had a format (#456). +fn pickle_tracer_before_format(tracer: &Tracer, out: &mut Vec) { + match tracer { + Tracer::Log(log) => { + 0u16.pickle(out); + log.path.pickle(out); + log.prefix.pickle(out); + log.rotate.pickle(out); + log.ansi.pickle(out); + log.multiline.pickle(out); + log.enable.pickle(out); + log.level.pickle(out); + log.lossy.pickle(out); + log.events.pickle(out); + log.events_policy.pickle(out); + } + other => other.pickle(out), + } +} + fn read(object: ObjectType, bytes: &[u8]) -> ObjectInner { ObjectInner::unpickle(object, &mut PickledStream::new(bytes).unwrap()).unwrap() } @@ -77,7 +97,7 @@ fn oidc_directories_saved_before_the_opaque_token_policy_keep_userinfo() { bootstrap.in_memory_store.pickle(&mut bytes); 3u16.pickle(&mut bytes); pickle_without_opaque_tokens(&saved, &mut bytes); - bootstrap.tracer.pickle(&mut bytes); + pickle_tracer_before_format(&bootstrap.tracer, &mut bytes); bootstrap.dns_server.pickle(&mut bytes); assert_eq!( read(ObjectType::Bootstrap, &bytes), @@ -219,3 +239,67 @@ fn logos_saved_before_dark_mode_read_back_without_a_dark_logo() { ObjectInner::Enterprise(enterprise) ); } + +// #456: Log tracers saved before the format setting read back as text, in a +// tracer of their own and in the bootstrap, where more fields follow the +// tracer; saved now, a JSON one reads back as JSON +#[test] +fn log_tracers_saved_before_the_format_read_back_as_text() { + let log = TracerLog { + path: "/var/log/inbuxa".to_string(), + prefix: "inbuxa.log".to_string(), + ansi: false, + ..Default::default() + }; + assert_eq!(log.format, LogFormat::Text); + let tracer = Tracer::Log(log.clone()); + + // x:Tracer, version 0 + let mut bytes = vec![0u8]; + pickle_tracer_before_format(&tracer, &mut bytes); + assert_eq!( + read(ObjectType::Tracer, &bytes), + ObjectInner::Tracer(tracer.clone()) + ); + + // x:Bootstrap, version 2 + let bootstrap = Bootstrap { + server_hostname: "mx.example.org".to_string(), + tracer: tracer.clone(), + ..Default::default() + }; + let mut bytes = vec![2u8]; + bootstrap.server_hostname.pickle(&mut bytes); + bootstrap.default_domain.pickle(&mut bytes); + bootstrap.request_tls_certificate.pickle(&mut bytes); + bootstrap.generate_dkim_keys.pickle(&mut bytes); + bootstrap.data_store.pickle(&mut bytes); + bootstrap.blob_store.pickle(&mut bytes); + bootstrap.search_store.pickle(&mut bytes); + bootstrap.in_memory_store.pickle(&mut bytes); + bootstrap.directory.pickle(&mut bytes); + pickle_tracer_before_format(&bootstrap.tracer, &mut bytes); + bootstrap.dns_server.pickle(&mut bytes); + assert_eq!( + read(ObjectType::Bootstrap, &bytes), + ObjectInner::Bootstrap(bootstrap.clone()) + ); + + // Saved now + let tracer = Tracer::Log(TracerLog { + format: LogFormat::Json, + ..log + }); + let bootstrap = Bootstrap { + tracer: tracer.clone(), + ..bootstrap + }; + assert_eq!( + read(ObjectType::Tracer, &tracer.to_pickled_vec()), + ObjectInner::Tracer(tracer) + ); + assert_eq!( + read(ObjectType::Bootstrap, &bootstrap.to_pickled_vec()), + ObjectInner::Bootstrap(bootstrap) + ); +} diff --git a/crates/registry/src/schema/enums.rs b/crates/registry/src/schema/enums.rs index e03f4045..c35e5b06 100644 --- a/crates/registry/src/schema/enums.rs +++ b/crates/registry/src/schema/enums.rs @@ -1272,6 +1272,15 @@ pub enum LogRotateFrequency { Never = 3, } +// inbuxa: #456, how a Log tracer writes each entry +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +#[repr(u16)] +pub enum LogFormat { + #[default] + Text = 0, + Json = 1, +} + #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] #[repr(u16)] pub enum LookupStoreType { diff --git a/crates/registry/src/schema/enums_impl.rs b/crates/registry/src/schema/enums_impl.rs index f0eee361..868df891 100644 --- a/crates/registry/src/schema/enums_impl.rs +++ b/crates/registry/src/schema/enums_impl.rs @@ -5522,6 +5522,57 @@ impl<'de> serde::Deserialize<'de> for Locale { } } +// inbuxa: #456 +impl EnumImpl for LogFormat { + fn parse(value: &str) -> Option { + hashify::tiny_map! { + value.as_bytes(), + b"text" => LogFormat::Text, + b"json" => LogFormat::Json, + } + } + + fn as_str(&self) -> &'static str { + match self { + LogFormat::Text => "text", + LogFormat::Json => "json", + } + } + + fn to_id(&self) -> u16 { + *self as u16 + } + + fn from_id(id: u16) -> Option { + match id { + 0 => Some(LogFormat::Text), + 1 => Some(LogFormat::Json), + _ => None, + } + } + + const COUNT: usize = 2; +} + +impl serde::Serialize for LogFormat { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> serde::Deserialize<'de> for LogFormat { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = Cow::::deserialize(deserializer)?; + Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[])) + } +} + impl EnumImpl for LogRotateFrequency { fn parse(value: &str) -> Option { hashify::tiny_map! { diff --git a/crates/registry/src/schema/structs.rs b/crates/registry/src/schema/structs.rs index c1592b13..f5622376 100644 --- a/crates/registry/src/schema/structs.rs +++ b/crates/registry/src/schema/structs.rs @@ -6164,6 +6164,9 @@ pub struct TracerLog { pub events: Map, #[serde(rename = "eventsPolicy")] pub events_policy: EventPolicy, + // inbuxa: #456 + #[serde(rename = "format")] + pub format: LogFormat, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] diff --git a/crates/registry/src/schema/structs_impl.rs b/crates/registry/src/schema/structs_impl.rs index 3bc1f29a..eec24487 100644 --- a/crates/registry/src/schema/structs_impl.rs +++ b/crates/registry/src/schema/structs_impl.rs @@ -4070,8 +4070,8 @@ impl RegistryJsonPropertyPatch for BlockedIp { impl ObjectImpl for Bootstrap { const FLAGS: u64 = OBJ_SINGLETON; // inbuxa: 2, as x:Directory: OidcDirectory gained fields at version 2 - // of both - const VERSION: u8 = 2; + // of both. 3, as x:Tracer: a Log tracer's format (#456) + const VERSION: u8 = 3; const OBJECT: ObjectType = ObjectType::Bootstrap; fn validate(&self, errors: &mut Vec) -> bool { @@ -46428,7 +46428,9 @@ impl RegistryJsonPropertyPatch for TraceValueUnsignedInt { impl ObjectImpl for Tracer { const FLAGS: u64 = 0; - const VERSION: u8 = 0; + // inbuxa: 3, as x:Bootstrap, which holds a tracer too: a Log tracer's + // format (#456) is stored from version 3 of both + const VERSION: u8 = 3; const OBJECT: ObjectType = ObjectType::Tracer; fn validate(&self, errors: &mut Vec) -> bool { @@ -46665,6 +46667,7 @@ impl Pickle for TracerLog { self.lossy.pickle(out); self.events.pickle(out); self.events_policy.pickle(out); + self.format.pickle(out); } fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { @@ -46679,6 +46682,10 @@ impl Pickle for TracerLog { this.lossy = Pickle::unpickle(stream)?; this.events = Pickle::unpickle(stream)?; this.events_policy = Pickle::unpickle(stream)?; + // inbuxa: #456, stored from version 3 of x:Tracer and x:Bootstrap + if stream.version() >= 3 { + this.format = Pickle::unpickle(stream)?; + } Some(this) } } @@ -46696,13 +46703,14 @@ impl Default for TracerLog { lossy: false, events: Default::default(), events_policy: EventPolicy::Exclude, + format: LogFormat::Text, } } } impl IntoValue for TracerLog { fn into_value(self) -> JmapValue<'static> { - let mut map = jmap_tools::Map::with_capacity(12); + let mut map = jmap_tools::Map::with_capacity(13); map.insert_unchecked(Property::Path, self.path.into_value()); map.insert_unchecked(Property::Prefix, self.prefix.into_value()); map.insert_unchecked(Property::Rotate, self.rotate.into_value()); @@ -46713,6 +46721,7 @@ impl IntoValue for TracerLog { map.insert_unchecked(Property::Lossy, self.lossy.into_value()); map.insert_unchecked(Property::Events, self.events.into_value()); map.insert_unchecked(Property::EventsPolicy, self.events_policy.into_value()); + map.insert_unchecked(Property::Format, self.format.into_value()); JmapValue::Object(map) } } @@ -46738,6 +46747,7 @@ impl RegistryJsonPropertyPatch for TracerLog { Some(Property::Lossy) => self.lossy.patch(pointer, value), Some(Property::Events) => self.events.patch(pointer, value), Some(Property::EventsPolicy) => self.events_policy.patch(pointer, value), + Some(Property::Format) => self.format.patch(pointer, value), Some(Property::Type) => Ok(MaybeUnpatched::Unpatched { property: Property::Type, value, diff --git a/crates/trc/src/serializers/json.rs b/crates/trc/src/serializers/json.rs index 96f79f4e..a61ac417 100644 --- a/crates/trc/src/serializers/json.rs +++ b/crates/trc/src/serializers/json.rs @@ -2,6 +2,8 @@ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + * + * Modified by Coffey Labs in 2026 for INBUXA. */ use crate::{Error, Event, EventDetails, EventType, Key, MetricType, Value}; @@ -123,6 +125,47 @@ impl>> Serialize for JsonEventSerializer { } } +/// inbuxa (#456): one entry of a Log tracer writing JSON lines: +/// `{"timestamp", "level", "event", "description", "data"}`, where `data` +/// holds the event's keys (and its span's) as typed values: nested events as +/// objects, lists as arrays. +pub struct JsonLogLine<'x>(pub &'x Event); + +impl Serialize for JsonLogLine<'_> { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let event = self.0; + let mut map = serializer.serialize_map(Some(5))?; + map.serialize_entry( + "timestamp", + &DateTime::from_timestamp(event.inner.timestamp as i64).to_rfc3339(), + )?; + map.serialize_entry("level", &event.inner.level.as_str().to_ascii_lowercase())?; + map.serialize_entry("event", event.inner.typ.as_str())?; + map.serialize_entry("description", event.inner.typ.description())?; + map.serialize_entry( + "data", + &JsonEventSerializer { + inner: Keys { + keys: event.keys.as_slice(), + span_keys: event + .inner + .span + .as_ref() + .map(|s| &s.keys[..]) + .unwrap_or(&[]), + }, + with_spans: false, + with_description: false, + with_id: false, + }, + )?; + map.end() + } +} + impl Serialize for JsonEventSerializer> { fn serialize(&self, serializer: S) -> Result where diff --git a/docs/spec/features/monitoring.md b/docs/spec/features/monitoring.md index 1dc7c777..ea625b30 100644 --- a/docs/spec/features/monitoring.md +++ b/docs/spec/features/monitoring.md @@ -252,6 +252,17 @@ unchanged. raised. Both list views show it between To and Size. It isn't in the `text` search keywords. Like every other trace field it is kept for `holdTracesFor`, and only the superuser set can read it. +- **MON-43.** (#456) A Log tracer has a `format`: `text` (the default, as + upstream) or `json`, one object per line: + `{"timestamp", "level", "event", "description", "data"}`, where `timestamp` + is RFC 3339, `level` is lowercase, and `data` holds the event's keys and its + span's as typed values (nested events as objects, lists as arrays). `ansi` + and `multiline` apply to text only. Stored at `x:Tracer` and `x:Bootstrap` + version 3; a tracer saved before reads back as `text`. `x:Log/get` and + `x:Log/query`, with every filter, read both formats, a mix of them too (a + tracer switched over keeps its older text lines). A JSON entry's `details` + is its `data` as a JSON object, `""` when it has none; a text entry's stays + the `key = value` text. ### Retention and growth @@ -579,7 +590,7 @@ they cover tests 10 and 12 (`tracing.rs`), 13 (`metrics.rs`) and 18 (`alerts.rs`). Tests 1, 3, 5 to 9, 11, 14 to 17, 19 to 21, 23 and 24 pass as `tests/src/system/monitoring.rs`. -- **MON-1 to MON-42:** built, with the limits below. +- **MON-1 to MON-43:** built, with the limits below. - **Test 26 (compat)** is written as `monitoring_compat`, ignored, and unrun until a copy of INBUXA's data is provided. It checks observed 1, reads the old history without an error, and purges it. It deletes history, so only diff --git a/docs/spec/features/personal-data-catalog.md b/docs/spec/features/personal-data-catalog.md index f2903e42..da3d2635 100644 --- a/docs/spec/features/personal-data-catalog.md +++ b/docs/spec/features/personal-data-catalog.md @@ -134,7 +134,7 @@ These live in inbuxa's own key space (`SUBSPACE_INBUXA`) unless noted. | Stored traces (`x:Trace`) | network (IP, EHLO), identifier (envelope addresses, account names), metadata (message-ids, results) | correspondent, holder | `x:TracingStore`; `x:EventTracingLevel` | `x:DataRetention.holdTracesFor` (unset = unbounded) | data store, or a remote PostgreSQL/MySQL/FDB | server (records name accounts) | `common/src/telemetry/tracers/store.rs` `spawn_store_tracer`; purge `purge_spans` | **collected** (first boot inserts `TracingStore::Default` although the struct default is off); 30 d | | Trace search index | same keywords, lowercased | as above | `x:Search.indexTelemetry`, `indexTracingFields` | purged with the traces | search store | server | `index.rs` `trace_search_document` | indexed | | Stored metrics | none personal (aggregates) | — | `x:MetricsStore`, `x:Metrics` | `holdMetricsFor` | data store | server | `telemetry/metrics/store.rs` | collected, 90 d | -| Log file | network (**client IP on every in-session line**, from the span), identifier (MAIL FROM, RCPT TO, account names), metadata (message-ids). No subject key exists; bodies only in trace-level raw-I/O events | correspondent, holder, administrator | `x:TracerLog` (`level`, `events`, `path`, `rotate`) | **unbounded: rotated daily, never deleted** | log file (`/var/log/inbuxa`); read back as `x:Log` and fed to Explain | server | `common/src/telemetry/tracers/log.rs` | **collected, info level** | +| Log file | network (**client IP on every in-session line**, from the span), identifier (MAIL FROM, RCPT TO, account names), metadata (message-ids). No subject key exists; bodies only in trace-level raw-I/O events | correspondent, holder, administrator | `x:TracerLog` (`level`, `events`, `path`, `rotate`, `format`: text or JSON lines) | **unbounded: rotated daily, never deleted** | log file (`/var/log/inbuxa`); read back as `x:Log` and fed to Explain | server | `common/src/telemetry/tracers/log.rs` | **collected, info level** | | Console / journald tracers | as the log file | as above | `x:Tracer` Stdout/Journal | the host's journal policy | stdout / journald (leaves only if the host forwards it) | server | `tracers/stdout.rs`, `journald.rs` | none (recovery mode adds a console tracer) | | OpenTelemetry tracer | as the log file, every event and span key | as above | `x:Tracer` OtelHttp/OtelGrpc (`level`, `events`, `endpoint`) | receiver's | **external** | server | `tracers/otel.rs` | none | | Webhooks | as the log file; **can include raw SMTP input (message content) and model replies** — see [Findings](#findings) 1 | as above | `x:WebHook` (`events`, `eventsPolicy`, `url`) | receiver's; retried up to `discardAfter` | **external** | server | `telemetry/webhooks/mod.rs` `post_webhook_events` | none | diff --git a/resources/schema/schema.json.gz b/resources/schema/schema.json.gz index 1f6a80ca..9b0cb7a9 100644 Binary files a/resources/schema/schema.json.gz and b/resources/schema/schema.json.gz differ diff --git a/resources/schema/schema.json.sha256 b/resources/schema/schema.json.sha256 index fb5ebcb1..8ef9f0b9 100644 --- a/resources/schema/schema.json.sha256 +++ b/resources/schema/schema.json.sha256 @@ -1 +1 @@ -k2_7GSOn4Rjf0ktF3x3Z88Z1u4hUHT_I_gEoRH_viO8 \ No newline at end of file +3kPB2eR8b9fyGhyDoSsRlB2AE3HhJrOzVnsdCpoURJY diff --git a/tests/src/system/tracer_reload.rs b/tests/src/system/tracer_reload.rs index 9a90eef8..a2568ded 100644 --- a/tests/src/system/tracer_reload.rs +++ b/tests/src/system/tracer_reload.rs @@ -14,7 +14,7 @@ use crate::utils::{ }; use registry::{ schema::{ - enums::{EventPolicy, LogRotateFrequency, TracingLevel}, + enums::{EventPolicy, LogFormat, LogRotateFrequency, TracingLevel}, prelude::ObjectType, structs::{Expression, MtaStageAuth, Tracer, TracerLog}, }, @@ -92,6 +92,7 @@ async fn test_log_tracer_moves(test: &TestServer) { lossy: false, events: Map::new(vec![EventType::Server(ServerEvent::Licensing)]), events_policy: EventPolicy::Include, + format: LogFormat::Text, })]) .await; assert_applied(&response); @@ -151,6 +152,34 @@ async fn test_log_tracer_moves(test: &TestServer) { "no event of the stream reached the new file" ); + // inbuxa (#456): switched to JSON lines, the tracer writes one object + // per line after the text it wrote before + let response = admin + .registry_update(ObjectType::Tracer, [(tracer_id, json!({"format": "json"}))]) + .await; + assert_applied(&response); + tokio::time::sleep(Duration::from_millis(200)).await; + emit("marker-json \"quoted\""); + wait_for(&new_file, "marker-json").await; + let text = read(&new_file); + let line = text + .lines() + .find(|line| line.contains("marker-json")) + .unwrap(); + let entry: serde_json::Value = serde_json::from_str(line).unwrap_or_else(|err| { + panic!("not a JSON line ({err}): {line}"); + }); + assert_eq!(entry["event"], "server.licensing", "{entry}"); + assert_eq!( + entry["data"]["details"], "marker-json \"quoted\"", + "{entry}" + ); + assert!( + text.lines() + .any(|line| line.contains("marker-after") && !line.starts_with('{')), + "the text written before stays text" + ); + // Removing the tracer stops it let response = admin .registry_destroy(ObjectType::Tracer, [tracer_id])