Log tracer: a JSON line format #462

Merged
jcoffey-dev merged 1 commits from feat/456-json-log-format into main 2026-11-18 22:27:09 +00:00
14 changed files with 548 additions and 18 deletions

No files matched your search

+4 -1
View File
@@ -17,7 +17,7 @@ use opentelemetry_otlp::{
use opentelemetry_sdk::{Resource, metrics::Temporality};
use opentelemetry_semantic_conventions::resource::SERVICE_VERSION;
use registry::schema::{
enums::{EventPolicy, LogRotateFrequency},
enums::{EventPolicy, LogFormat, LogRotateFrequency},
prelude::ObjectType,
structs::{self, EventTracingLevel, MetricsPrometheus, Tracer, WebHook},
};
@@ -87,6 +87,8 @@ pub struct LogTracer {
pub rotate: RotationStrategy,
pub ansi: bool,
pub multiline: bool,
// inbuxa: #456, one JSON object per line instead of text
pub json: bool,
}
#[derive(Debug)]
@@ -196,6 +198,7 @@ impl Tracers {
},
ansi: tracer.ansi,
multiline: tracer.multiline,
json: tracer.format == LogFormat::Json,
})
}
Tracer::Stdout(tracer) if tracer.enable => {
+106 -5
View File
@@ -13,9 +13,66 @@ use crate::config::telemetry::{LogTracer, RotationStrategy};
use mail_parser::DateTime;
use tokio::{
fs::{File, OpenOptions},
io::BufWriter,
io::{AsyncWriteExt, BufWriter},
};
use trc::{TelemetryEvent, ipc::subscriber::SubscriberBuilder, serializers::text::FmtWriter};
use trc::{
Event, EventDetails, TelemetryEvent,
ipc::subscriber::SubscriberBuilder,
serializers::{json::JsonLogLine, text::FmtWriter},
};
// inbuxa (#456): a Log tracer writes text, as upstream, or one JSON object
// per line.
enum LogWriter {
Text(FmtWriter<BufWriter<File>>),
Json(BufWriter<File>, Vec<u8>),
}
impl LogWriter {
fn new(writer: BufWriter<File>, settings: &LogTracer) -> Self {
if settings.json {
LogWriter::Json(writer, Vec::with_capacity(512))
} else {
LogWriter::Text(
FmtWriter::new(writer)
.with_ansi(settings.ansi)
.with_multiline(settings.multiline),
)
}
}
async fn write(&mut self, event: &Event<EventDetails>) -> std::io::Result<()> {
match self {
LogWriter::Text(writer) => writer.write(event).await,
LogWriter::Json(writer, line) => {
line.clear();
json_log_line(event, line)?;
writer.write_all(line).await
}
}
}
async fn flush(&mut self) -> std::io::Result<()> {
match self {
LogWriter::Text(writer) => writer.flush().await,
LogWriter::Json(writer, _) => writer.flush().await,
}
}
fn update_writer(&mut self, new: BufWriter<File>) {
match self {
LogWriter::Text(writer) => writer.update_writer(new),
LogWriter::Json(writer, _) => *writer = new,
}
}
}
/// inbuxa (#456): an event as one line of JSON, newline included.
pub(crate) fn json_log_line(event: &Event<EventDetails>, out: &mut Vec<u8>) -> std::io::Result<()> {
serde_json::to_writer(&mut *out, &JsonLogLine(event)).map_err(std::io::Error::other)?;
out.push(b'\n');
Ok(())
}
// inbuxa: when a Log tracer is started over on the same files (its rotation
// or format changed), the new one waits for the old one to write what it
@@ -39,9 +96,7 @@ pub(crate) fn spawn_log_tracer(builder: SubscriberBuilder, settings: LogTracer)
}
if let Some(writer) = settings.build_writer().await {
let mut buf = FmtWriter::new(writer)
.with_ansi(settings.ansi)
.with_multiline(settings.multiline);
let mut buf = LogWriter::new(writer, &settings);
let mut roatation_timestamp = settings.next_rotation();
while let Some(events) = rx.recv().await {
@@ -159,3 +214,49 @@ impl LogTracer {
}
}
}
#[cfg(test)]
mod tests {
use super::json_log_line;
use trc::{Event, EventDetails, EventType, Key, Level, SmtpEvent, Value};
// inbuxa (#456): a JSON line is one object with typed fields
#[test]
fn json_line_is_one_typed_object() {
let event = Event {
inner: EventDetails {
typ: EventType::Smtp(SmtpEvent::ConnectionStart),
timestamp: 1_794_996_000,
level: Level::Info,
span: None,
},
keys: vec![
(Key::RemoteIp, Value::String("192.0.2.1".into())),
(Key::RemotePort, Value::UInt(25)),
(
Key::Details,
Value::String("a \"quoted\" word\nand a line".into()),
),
(
Key::Contents,
Value::Array(vec![Value::Bool(true), Value::Int(-1)]),
),
],
};
let mut out = Vec::new();
json_log_line(&event, &mut out).unwrap();
let text = String::from_utf8(out).unwrap();
assert!(text.ends_with('\n'));
assert_eq!(text.matches('\n').count(), 1, "one line: {text}");
let json: serde_json::Value = serde_json::from_str(&text).unwrap();
assert_eq!(json["timestamp"], "2026-11-18T10:00:00Z");
assert_eq!(json["level"], "info");
assert_eq!(json["event"], "smtp.connection-start");
assert!(json["description"].as_str().is_some_and(|d| !d.is_empty()));
assert_eq!(json["data"]["remoteIp"], "192.0.2.1");
assert_eq!(json["data"]["remotePort"], 25);
assert_eq!(json["data"]["details"], "a \"quoted\" word\nand a line");
assert_eq!(json["data"]["contents"], serde_json::json!([true, -1]));
}
}
+187 -1
View File
@@ -252,6 +252,17 @@ struct LogHeader<'x> {
}
fn log_header(line: &str) -> Option<LogHeader<'_>> {
// inbuxa (#456): a Log tracer writing JSON lines
if line.starts_with('{') {
let entry = serde_json::from_str::<JsonLogHead<'_>>(line).ok()?;
return Some(LogHeader {
timestamp: DateTime::parse_from_rfc3339(entry.timestamp)
.ok()?
.timestamp(),
level: TracingLevel::parse(&entry.level.to_ascii_lowercase()).map(|l| l as u8),
event: entry.event,
});
}
if !line.as_bytes().first()?.is_ascii_digit() {
return None;
}
@@ -409,7 +420,53 @@ pub(crate) fn read_log_entries(
Ok(entries)
}
/// inbuxa (#456): the fields of a JSON log line that a query filters on.
#[derive(serde::Deserialize)]
struct JsonLogHead<'x> {
#[serde(borrow)]
timestamp: &'x str,
#[serde(borrow)]
level: &'x str,
#[serde(borrow)]
event: &'x str,
}
/// inbuxa (#456): a JSON log line, its `data` handed back as a JSON object
/// in `details`, so a reader gets the fields exactly as they were written.
#[derive(serde::Deserialize)]
struct JsonLogEntry<'x> {
#[serde(borrow)]
timestamp: &'x str,
#[serde(borrow)]
level: &'x str,
#[serde(borrow)]
event: &'x str,
#[serde(default)]
data: serde_json::Map<String, serde_json::Value>,
}
fn log_from_json_line(line: &str) -> Option<Log> {
let entry = serde_json::from_str::<JsonLogEntry<'_>>(line.trim_end()).ok()?;
Some(Log {
timestamp: UTCDateTime::from_timestamp(
DateTime::parse_from_rfc3339(entry.timestamp)
.ok()?
.timestamp(),
),
level: TracingLevel::parse(&entry.level.to_ascii_lowercase()).unwrap_or(TracingLevel::Info),
event: EventType::parse(entry.event)?,
details: if entry.data.is_empty() {
String::new()
} else {
serde_json::Value::Object(entry.data).to_string()
},
})
}
fn log_from_line(line: &str) -> Option<Log> {
if line.starts_with('{') {
return log_from_json_line(line);
}
let line = strip_ansi(line);
let (timestamp, rest) = line.split_once(' ')?;
let timestamp = DateTime::parse_from_rfc3339(timestamp).ok()?;
@@ -636,7 +693,7 @@ impl<R: Read + Seek> Iterator for RevLines<R> {
#[cfg(test)]
mod tests {
use super::{LogFilter, read_log_offsets};
use super::{LogFilter, read_log_entries, read_log_offsets};
#[test]
fn limit_zero_reads_no_entries() {
@@ -702,4 +759,133 @@ mod tests {
assert_eq!(window, 2);
assert_eq!(after, 2);
}
// inbuxa (#456): files written as JSON lines are queried and read like
// text ones, and a mix of both (a tracer switched over) reads in order
#[test]
fn reads_json_lines() {
let dir = std::env::temp_dir().join(format!("inbuxa-log-json-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(
dir.join("inbuxa.log.2026-11-18"),
concat!(
"2026-11-18T10:00:00Z INFO SMTP connection started (smtp.connection-start) remoteIp = 192.0.2.1\n",
r#"{"timestamp":"2026-11-18T10:00:01Z","level":"warn","event":"store.data-corruption","description":"Data corruption","data":{"details":"a \"quoted\" word, with a comma"}}"#,
"\n",
r#"{"timestamp":"2026-11-18T10:00:02Z","level":"error","event":"imap.connection-end","description":"x","data":{}}"#,
"\n",
r#"{"timestamp":"2026-11-18T10:00:03Z","level":"debug","event":"smtp.rcpt-to","description":"y","data":{"to":["[email protected]"],"size":12}}"#,
"\n",
),
)
.unwrap();
let ids = |filter: LogFilter| read_log_offsets(&dir, &filter, 0, 10).unwrap();
let all = ids(LogFilter::default());
let at_least_warn = ids(LogFilter {
level: Some(1),
..Default::default()
});
let smtp = ids(LogFilter {
event: Some("smtp".into()),
..Default::default()
});
let window = ids(LogFilter {
from: 1_794_996_001,
to: 1_794_996_002,
..Default::default()
});
let text = ids(LogFilter {
text: Some("comma".into()),
..Default::default()
});
let entries = read_log_entries(&dir, Some(all.clone()), 10).unwrap();
std::fs::remove_dir_all(&dir).unwrap();
assert_eq!(all.len(), 4);
assert_eq!(at_least_warn.len(), 2, "warn and error");
assert_eq!(smtp.len(), 2, "smtp.* only");
assert_eq!(window.len(), 2);
assert_eq!(text.len(), 1);
// Newest first
let events = entries
.iter()
.map(|(_, log)| log.event.as_str())
.collect::<Vec<_>>();
assert_eq!(
events,
[
"smtp.rcpt-to",
"imap.connection-end",
"store.data-corruption",
"smtp.connection-start"
]
);
let details = |i: usize| entries[i].1.details.as_str();
let rcpt: serde_json::Value = serde_json::from_str(details(0)).unwrap();
assert_eq!(
rcpt,
serde_json::json!({"to": ["[email protected]"], "size": 12})
);
assert_eq!(details(1), "", "no fields, no details");
let corruption: serde_json::Value = serde_json::from_str(details(2)).unwrap();
assert_eq!(corruption["details"], "a \"quoted\" word, with a comma");
assert_eq!(details(3), "remoteIp = 192.0.2.1");
assert_eq!(entries[2].1.level as u8, 1);
}
// inbuxa (#456): what the Log tracer writes as JSON reads back exactly
#[test]
fn json_tracer_output_reads_back() {
use trc::{Event, EventDetails, EventType, Key, Level, SmtpEvent, Value};
let event = Event {
inner: EventDetails {
typ: EventType::Smtp(SmtpEvent::RcptTo),
timestamp: 1_794_996_000,
level: Level::Debug,
span: Some(std::sync::Arc::new(Event {
inner: EventDetails {
typ: EventType::Smtp(SmtpEvent::ConnectionStart),
timestamp: 1_794_995_999,
level: Level::Info,
span: None,
},
keys: vec![(Key::RemoteIp, Value::String("192.0.2.1".into()))],
})),
},
keys: vec![(Key::To, Value::String("a \"b\", [email protected]".into()))],
};
let mut line = serde_json::to_string(&trc::serializers::json::JsonLogLine(&event)).unwrap();
line.push('\n');
let dir = std::env::temp_dir().join(format!("inbuxa-log-rt-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join("inbuxa.log"), &line).unwrap();
let ids = read_log_offsets(
&dir,
&LogFilter {
level: Some(3),
event: Some("smtp.rcpt-to".into()),
..Default::default()
},
0,
10,
)
.unwrap();
let entries = read_log_entries(&dir, Some(ids), 10).unwrap();
std::fs::remove_dir_all(&dir).unwrap();
assert_eq!(entries.len(), 1);
let log = &entries[0].1;
assert_eq!(log.event, EventType::Smtp(SmtpEvent::RcptTo));
assert_eq!(log.level as u8, 3);
assert_eq!(log.timestamp.timestamp(), 1_794_996_000);
let data: serde_json::Value = serde_json::from_str(&log.details).unwrap();
assert_eq!(
data,
serde_json::json!({"to": "a \"b\", [email protected]", "remoteIp": "192.0.2.1"})
);
}
}
+87 -3
View File
@@ -9,11 +9,11 @@
use crate::{
pickle::{Pickle, PickledStream},
schema::{
enums::{MtaRouteType, OidcOpaqueTokens},
enums::{LogFormat, MtaRouteType, OidcOpaqueTokens},
prelude::{ObjectImpl, ObjectInner, ObjectType},
structs::{
Bootstrap, Directory, DirectoryBootstrap, Domain, Enterprise, MtaRoute, MtaRouteCommon,
MtaRouteInbuxaRelay, OidcDirectory, Tenant,
MtaRouteInbuxaRelay, OidcDirectory, Tenant, Tracer, TracerLog,
},
},
types::{EnumImpl, map::Map},
@@ -32,6 +32,26 @@ fn pickle_without_opaque_tokens(directory: &OidcDirectory, out: &mut Vec<u8>) {
directory.member_tenant_id.pickle(out);
}
/// A tracer as it was stored before a Log tracer had a format (#456).
fn pickle_tracer_before_format(tracer: &Tracer, out: &mut Vec<u8>) {
match tracer {
Tracer::Log(log) => {
0u16.pickle(out);
log.path.pickle(out);
log.prefix.pickle(out);
log.rotate.pickle(out);
log.ansi.pickle(out);
log.multiline.pickle(out);
log.enable.pickle(out);
log.level.pickle(out);
log.lossy.pickle(out);
log.events.pickle(out);
log.events_policy.pickle(out);
}
other => other.pickle(out),
}
}
fn read(object: ObjectType, bytes: &[u8]) -> ObjectInner {
ObjectInner::unpickle(object, &mut PickledStream::new(bytes).unwrap()).unwrap()
}
@@ -77,7 +97,7 @@ fn oidc_directories_saved_before_the_opaque_token_policy_keep_userinfo() {
bootstrap.in_memory_store.pickle(&mut bytes);
3u16.pickle(&mut bytes);
pickle_without_opaque_tokens(&saved, &mut bytes);
bootstrap.tracer.pickle(&mut bytes);
pickle_tracer_before_format(&bootstrap.tracer, &mut bytes);
bootstrap.dns_server.pickle(&mut bytes);
assert_eq!(
read(ObjectType::Bootstrap, &bytes),
@@ -219,3 +239,67 @@ fn logos_saved_before_dark_mode_read_back_without_a_dark_logo() {
ObjectInner::Enterprise(enterprise)
);
}
// #456: Log tracers saved before the format setting read back as text, in a
// tracer of their own and in the bootstrap, where more fields follow the
// tracer; saved now, a JSON one reads back as JSON
#[test]
fn log_tracers_saved_before_the_format_read_back_as_text() {
let log = TracerLog {
path: "/var/log/inbuxa".to_string(),
prefix: "inbuxa.log".to_string(),
ansi: false,
..Default::default()
};
assert_eq!(log.format, LogFormat::Text);
let tracer = Tracer::Log(log.clone());
// x:Tracer, version 0
let mut bytes = vec![0u8];
pickle_tracer_before_format(&tracer, &mut bytes);
assert_eq!(
read(ObjectType::Tracer, &bytes),
ObjectInner::Tracer(tracer.clone())
);
// x:Bootstrap, version 2
let bootstrap = Bootstrap {
server_hostname: "mx.example.org".to_string(),
tracer: tracer.clone(),
..Default::default()
};
let mut bytes = vec![2u8];
bootstrap.server_hostname.pickle(&mut bytes);
bootstrap.default_domain.pickle(&mut bytes);
bootstrap.request_tls_certificate.pickle(&mut bytes);
bootstrap.generate_dkim_keys.pickle(&mut bytes);
bootstrap.data_store.pickle(&mut bytes);
bootstrap.blob_store.pickle(&mut bytes);
bootstrap.search_store.pickle(&mut bytes);
bootstrap.in_memory_store.pickle(&mut bytes);
bootstrap.directory.pickle(&mut bytes);
pickle_tracer_before_format(&bootstrap.tracer, &mut bytes);
bootstrap.dns_server.pickle(&mut bytes);
assert_eq!(
read(ObjectType::Bootstrap, &bytes),
ObjectInner::Bootstrap(bootstrap.clone())
);
// Saved now
let tracer = Tracer::Log(TracerLog {
format: LogFormat::Json,
..log
});
let bootstrap = Bootstrap {
tracer: tracer.clone(),
..bootstrap
};
assert_eq!(
read(ObjectType::Tracer, &tracer.to_pickled_vec()),
ObjectInner::Tracer(tracer)
);
assert_eq!(
read(ObjectType::Bootstrap, &bootstrap.to_pickled_vec()),
ObjectInner::Bootstrap(bootstrap)
);
}
+9
View File
@@ -1272,6 +1272,15 @@ pub enum LogRotateFrequency {
Never = 3,
}
// inbuxa: #456, how a Log tracer writes each entry
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
#[repr(u16)]
pub enum LogFormat {
#[default]
Text = 0,
Json = 1,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
#[repr(u16)]
pub enum LookupStoreType {
+51
View File
@@ -5522,6 +5522,57 @@ impl<'de> serde::Deserialize<'de> for Locale {
}
}
// inbuxa: #456
impl EnumImpl for LogFormat {
fn parse(value: &str) -> Option<Self> {
hashify::tiny_map! {
value.as_bytes(),
b"text" => LogFormat::Text,
b"json" => LogFormat::Json,
}
}
fn as_str(&self) -> &'static str {
match self {
LogFormat::Text => "text",
LogFormat::Json => "json",
}
}
fn to_id(&self) -> u16 {
*self as u16
}
fn from_id(id: u16) -> Option<Self> {
match id {
0 => Some(LogFormat::Text),
1 => Some(LogFormat::Json),
_ => None,
}
}
const COUNT: usize = 2;
}
impl serde::Serialize for LogFormat {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.serialize_str(self.as_str())
}
}
impl<'de> serde::Deserialize<'de> for LogFormat {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let s = Cow::<str>::deserialize(deserializer)?;
Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[]))
}
}
impl EnumImpl for LogRotateFrequency {
fn parse(value: &str) -> Option<Self> {
hashify::tiny_map! {
+3
View File
@@ -6164,6 +6164,9 @@ pub struct TracerLog {
pub events: Map<trc::EventType>,
#[serde(rename = "eventsPolicy")]
pub events_policy: EventPolicy,
// inbuxa: #456
#[serde(rename = "format")]
pub format: LogFormat,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+14 -4
View File
@@ -4070,8 +4070,8 @@ impl RegistryJsonPropertyPatch for BlockedIp {
impl ObjectImpl for Bootstrap {
const FLAGS: u64 = OBJ_SINGLETON;
// inbuxa: 2, as x:Directory: OidcDirectory gained fields at version 2
// of both
const VERSION: u8 = 2;
// of both. 3, as x:Tracer: a Log tracer's format (#456)
const VERSION: u8 = 3;
const OBJECT: ObjectType = ObjectType::Bootstrap;
fn validate(&self, errors: &mut Vec<ValidationError>) -> bool {
@@ -46428,7 +46428,9 @@ impl RegistryJsonPropertyPatch for TraceValueUnsignedInt {
impl ObjectImpl for Tracer {
const FLAGS: u64 = 0;
const VERSION: u8 = 0;
// inbuxa: 3, as x:Bootstrap, which holds a tracer too: a Log tracer's
// format (#456) is stored from version 3 of both
const VERSION: u8 = 3;
const OBJECT: ObjectType = ObjectType::Tracer;
fn validate(&self, errors: &mut Vec<ValidationError>) -> bool {
@@ -46665,6 +46667,7 @@ impl Pickle for TracerLog {
self.lossy.pickle(out);
self.events.pickle(out);
self.events_policy.pickle(out);
self.format.pickle(out);
}
fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option<Self> {
@@ -46679,6 +46682,10 @@ impl Pickle for TracerLog {
this.lossy = Pickle::unpickle(stream)?;
this.events = Pickle::unpickle(stream)?;
this.events_policy = Pickle::unpickle(stream)?;
// inbuxa: #456, stored from version 3 of x:Tracer and x:Bootstrap
if stream.version() >= 3 {
this.format = Pickle::unpickle(stream)?;
}
Some(this)
}
}
@@ -46696,13 +46703,14 @@ impl Default for TracerLog {
lossy: false,
events: Default::default(),
events_policy: EventPolicy::Exclude,
format: LogFormat::Text,
}
}
}
impl IntoValue for TracerLog {
fn into_value(self) -> JmapValue<'static> {
let mut map = jmap_tools::Map::with_capacity(12);
let mut map = jmap_tools::Map::with_capacity(13);
map.insert_unchecked(Property::Path, self.path.into_value());
map.insert_unchecked(Property::Prefix, self.prefix.into_value());
map.insert_unchecked(Property::Rotate, self.rotate.into_value());
@@ -46713,6 +46721,7 @@ impl IntoValue for TracerLog {
map.insert_unchecked(Property::Lossy, self.lossy.into_value());
map.insert_unchecked(Property::Events, self.events.into_value());
map.insert_unchecked(Property::EventsPolicy, self.events_policy.into_value());
map.insert_unchecked(Property::Format, self.format.into_value());
JmapValue::Object(map)
}
}
@@ -46738,6 +46747,7 @@ impl RegistryJsonPropertyPatch for TracerLog {
Some(Property::Lossy) => self.lossy.patch(pointer, value),
Some(Property::Events) => self.events.patch(pointer, value),
Some(Property::EventsPolicy) => self.events_policy.patch(pointer, value),
Some(Property::Format) => self.format.patch(pointer, value),
Some(Property::Type) => Ok(MaybeUnpatched::Unpatched {
property: Property::Type,
value,
+43
View File
@@ -2,6 +2,8 @@
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*
* Modified by Coffey Labs in 2026 for INBUXA.
*/
use crate::{Error, Event, EventDetails, EventType, Key, MetricType, Value};
@@ -123,6 +125,47 @@ impl<T: AsRef<Event<EventDetails>>> Serialize for JsonEventSerializer<T> {
}
}
/// inbuxa (#456): one entry of a Log tracer writing JSON lines:
/// `{"timestamp", "level", "event", "description", "data"}`, where `data`
/// holds the event's keys (and its span's) as typed values: nested events as
/// objects, lists as arrays.
pub struct JsonLogLine<'x>(pub &'x Event<EventDetails>);
impl Serialize for JsonLogLine<'_> {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let event = self.0;
let mut map = serializer.serialize_map(Some(5))?;
map.serialize_entry(
"timestamp",
&DateTime::from_timestamp(event.inner.timestamp as i64).to_rfc3339(),
)?;
map.serialize_entry("level", &event.inner.level.as_str().to_ascii_lowercase())?;
map.serialize_entry("event", event.inner.typ.as_str())?;
map.serialize_entry("description", event.inner.typ.description())?;
map.serialize_entry(
"data",
&JsonEventSerializer {
inner: Keys {
keys: event.keys.as_slice(),
span_keys: event
.inner
.span
.as_ref()
.map(|s| &s.keys[..])
.unwrap_or(&[]),
},
with_spans: false,
with_description: false,
with_id: false,
},
)?;
map.end()
}
}
impl Serialize for JsonEventSerializer<Keys<'_>> {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
+12 -1
View File
@@ -252,6 +252,17 @@ unchanged.
raised. Both list views show it between To and Size. It isn't in the
`text` search keywords. Like every other trace field it is kept for
`holdTracesFor`, and only the superuser set can read it.
- **MON-43.** (#456) A Log tracer has a `format`: `text` (the default, as
upstream) or `json`, one object per line:
`{"timestamp", "level", "event", "description", "data"}`, where `timestamp`
is RFC 3339, `level` is lowercase, and `data` holds the event's keys and its
span's as typed values (nested events as objects, lists as arrays). `ansi`
and `multiline` apply to text only. Stored at `x:Tracer` and `x:Bootstrap`
version 3; a tracer saved before reads back as `text`. `x:Log/get` and
`x:Log/query`, with every filter, read both formats, a mix of them too (a
tracer switched over keeps its older text lines). A JSON entry's `details`
is its `data` as a JSON object, `""` when it has none; a text entry's stays
the `key = value` text.
### Retention and growth
@@ -579,7 +590,7 @@ they cover tests 10 and 12 (`tracing.rs`), 13 (`metrics.rs`) and 18
(`alerts.rs`). Tests 1, 3, 5 to 9, 11, 14 to 17, 19 to 21, 23 and 24 pass as
`tests/src/system/monitoring.rs`.
- **MON-1 to MON-42:** built, with the limits below.
- **MON-1 to MON-43:** built, with the limits below.
- **Test 26 (compat)** is written as `monitoring_compat`, ignored, and unrun
until a copy of INBUXA's data is provided. It checks observed 1, reads the
old history without an error, and purges it. It deletes history, so only
+1 -1
View File
@@ -134,7 +134,7 @@ These live in inbuxa's own key space (`SUBSPACE_INBUXA`) unless noted.
| Stored traces (`x:Trace`) | network (IP, EHLO), identifier (envelope addresses, account names), metadata (message-ids, results) | correspondent, holder | `x:TracingStore`; `x:EventTracingLevel` | `x:DataRetention.holdTracesFor` (unset = unbounded) | data store, or a remote PostgreSQL/MySQL/FDB | server (records name accounts) | `common/src/telemetry/tracers/store.rs` `spawn_store_tracer`; purge `purge_spans` | **collected** (first boot inserts `TracingStore::Default` although the struct default is off); 30 d |
| Trace search index | same keywords, lowercased | as above | `x:Search.indexTelemetry`, `indexTracingFields` | purged with the traces | search store | server | `index.rs` `trace_search_document` | indexed |
| Stored metrics | none personal (aggregates) | — | `x:MetricsStore`, `x:Metrics` | `holdMetricsFor` | data store | server | `telemetry/metrics/store.rs` | collected, 90 d |
| Log file | network (**client IP on every in-session line**, from the span), identifier (MAIL FROM, RCPT TO, account names), metadata (message-ids). No subject key exists; bodies only in trace-level raw-I/O events | correspondent, holder, administrator | `x:TracerLog` (`level`, `events`, `path`, `rotate`) | **unbounded: rotated daily, never deleted** | log file (`/var/log/inbuxa`); read back as `x:Log` and fed to Explain | server | `common/src/telemetry/tracers/log.rs` | **collected, info level** |
| Log file | network (**client IP on every in-session line**, from the span), identifier (MAIL FROM, RCPT TO, account names), metadata (message-ids). No subject key exists; bodies only in trace-level raw-I/O events | correspondent, holder, administrator | `x:TracerLog` (`level`, `events`, `path`, `rotate`, `format`: text or JSON lines) | **unbounded: rotated daily, never deleted** | log file (`/var/log/inbuxa`); read back as `x:Log` and fed to Explain | server | `common/src/telemetry/tracers/log.rs` | **collected, info level** |
| Console / journald tracers | as the log file | as above | `x:Tracer` Stdout/Journal | the host's journal policy | stdout / journald (leaves only if the host forwards it) | server | `tracers/stdout.rs`, `journald.rs` | none (recovery mode adds a console tracer) |
| OpenTelemetry tracer | as the log file, every event and span key | as above | `x:Tracer` OtelHttp/OtelGrpc (`level`, `events`, `endpoint`) | receiver's | **external** | server | `tracers/otel.rs` | none |
| Webhooks | as the log file; **can include raw SMTP input (message content) and model replies** — see [Findings](#findings) 1 | as above | `x:WebHook` (`events`, `eventsPolicy`, `url`) | receiver's; retried up to `discardAfter` | **external** | server | `telemetry/webhooks/mod.rs` `post_webhook_events` | none |
Binary file not shown.
+1 -1
View File
@@ -1 +1 @@
k2_7GSOn4Rjf0ktF3x3Z88Z1u4hUHT_I_gEoRH_viO8
3kPB2eR8b9fyGhyDoSsRlB2AE3HhJrOzVnsdCpoURJY
+30 -1
View File
@@ -14,7 +14,7 @@ use crate::utils::{
};
use registry::{
schema::{
enums::{EventPolicy, LogRotateFrequency, TracingLevel},
enums::{EventPolicy, LogFormat, LogRotateFrequency, TracingLevel},
prelude::ObjectType,
structs::{Expression, MtaStageAuth, Tracer, TracerLog},
},
@@ -92,6 +92,7 @@ async fn test_log_tracer_moves(test: &TestServer) {
lossy: false,
events: Map::new(vec![EventType::Server(ServerEvent::Licensing)]),
events_policy: EventPolicy::Include,
format: LogFormat::Text,
})])
.await;
assert_applied(&response);
@@ -151,6 +152,34 @@ async fn test_log_tracer_moves(test: &TestServer) {
"no event of the stream reached the new file"
);
// inbuxa (#456): switched to JSON lines, the tracer writes one object
// per line after the text it wrote before
let response = admin
.registry_update(ObjectType::Tracer, [(tracer_id, json!({"format": "json"}))])
.await;
assert_applied(&response);
tokio::time::sleep(Duration::from_millis(200)).await;
emit("marker-json \"quoted\"");
wait_for(&new_file, "marker-json").await;
let text = read(&new_file);
let line = text
.lines()
.find(|line| line.contains("marker-json"))
.unwrap();
let entry: serde_json::Value = serde_json::from_str(line).unwrap_or_else(|err| {
panic!("not a JSON line ({err}): {line}");
});
assert_eq!(entry["event"], "server.licensing", "{entry}");
assert_eq!(
entry["data"]["details"], "marker-json \"quoted\"",
"{entry}"
);
assert!(
text.lines()
.any(|line| line.contains("marker-after") && !line.starts_with('{')),
"the text written before stays text"
);
// Removing the tracer stops it
let response = admin
.registry_destroy(ObjectType::Tracer, [tracer_id])