Log tracer: a JSON line format #462
No files matched your search
@@ -17,7 +17,7 @@ use opentelemetry_otlp::{
|
||||
use opentelemetry_sdk::{Resource, metrics::Temporality};
|
||||
use opentelemetry_semantic_conventions::resource::SERVICE_VERSION;
|
||||
use registry::schema::{
|
||||
enums::{EventPolicy, LogRotateFrequency},
|
||||
enums::{EventPolicy, LogFormat, LogRotateFrequency},
|
||||
prelude::ObjectType,
|
||||
structs::{self, EventTracingLevel, MetricsPrometheus, Tracer, WebHook},
|
||||
};
|
||||
@@ -87,6 +87,8 @@ pub struct LogTracer {
|
||||
pub rotate: RotationStrategy,
|
||||
pub ansi: bool,
|
||||
pub multiline: bool,
|
||||
// inbuxa: #456, one JSON object per line instead of text
|
||||
pub json: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -196,6 +198,7 @@ impl Tracers {
|
||||
},
|
||||
ansi: tracer.ansi,
|
||||
multiline: tracer.multiline,
|
||||
json: tracer.format == LogFormat::Json,
|
||||
})
|
||||
}
|
||||
Tracer::Stdout(tracer) if tracer.enable => {
|
||||
|
||||
@@ -13,9 +13,66 @@ use crate::config::telemetry::{LogTracer, RotationStrategy};
|
||||
use mail_parser::DateTime;
|
||||
use tokio::{
|
||||
fs::{File, OpenOptions},
|
||||
io::BufWriter,
|
||||
io::{AsyncWriteExt, BufWriter},
|
||||
};
|
||||
use trc::{TelemetryEvent, ipc::subscriber::SubscriberBuilder, serializers::text::FmtWriter};
|
||||
use trc::{
|
||||
Event, EventDetails, TelemetryEvent,
|
||||
ipc::subscriber::SubscriberBuilder,
|
||||
serializers::{json::JsonLogLine, text::FmtWriter},
|
||||
};
|
||||
|
||||
// inbuxa (#456): a Log tracer writes text, as upstream, or one JSON object
|
||||
// per line.
|
||||
enum LogWriter {
|
||||
Text(FmtWriter<BufWriter<File>>),
|
||||
Json(BufWriter<File>, Vec<u8>),
|
||||
}
|
||||
|
||||
impl LogWriter {
|
||||
fn new(writer: BufWriter<File>, settings: &LogTracer) -> Self {
|
||||
if settings.json {
|
||||
LogWriter::Json(writer, Vec::with_capacity(512))
|
||||
} else {
|
||||
LogWriter::Text(
|
||||
FmtWriter::new(writer)
|
||||
.with_ansi(settings.ansi)
|
||||
.with_multiline(settings.multiline),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
async fn write(&mut self, event: &Event<EventDetails>) -> std::io::Result<()> {
|
||||
match self {
|
||||
LogWriter::Text(writer) => writer.write(event).await,
|
||||
LogWriter::Json(writer, line) => {
|
||||
line.clear();
|
||||
json_log_line(event, line)?;
|
||||
writer.write_all(line).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn flush(&mut self) -> std::io::Result<()> {
|
||||
match self {
|
||||
LogWriter::Text(writer) => writer.flush().await,
|
||||
LogWriter::Json(writer, _) => writer.flush().await,
|
||||
}
|
||||
}
|
||||
|
||||
fn update_writer(&mut self, new: BufWriter<File>) {
|
||||
match self {
|
||||
LogWriter::Text(writer) => writer.update_writer(new),
|
||||
LogWriter::Json(writer, _) => *writer = new,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// inbuxa (#456): an event as one line of JSON, newline included.
|
||||
pub(crate) fn json_log_line(event: &Event<EventDetails>, out: &mut Vec<u8>) -> std::io::Result<()> {
|
||||
serde_json::to_writer(&mut *out, &JsonLogLine(event)).map_err(std::io::Error::other)?;
|
||||
out.push(b'\n');
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// inbuxa: when a Log tracer is started over on the same files (its rotation
|
||||
// or format changed), the new one waits for the old one to write what it
|
||||
@@ -39,9 +96,7 @@ pub(crate) fn spawn_log_tracer(builder: SubscriberBuilder, settings: LogTracer)
|
||||
}
|
||||
|
||||
if let Some(writer) = settings.build_writer().await {
|
||||
let mut buf = FmtWriter::new(writer)
|
||||
.with_ansi(settings.ansi)
|
||||
.with_multiline(settings.multiline);
|
||||
let mut buf = LogWriter::new(writer, &settings);
|
||||
let mut roatation_timestamp = settings.next_rotation();
|
||||
|
||||
while let Some(events) = rx.recv().await {
|
||||
@@ -159,3 +214,49 @@ impl LogTracer {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::json_log_line;
|
||||
use trc::{Event, EventDetails, EventType, Key, Level, SmtpEvent, Value};
|
||||
|
||||
// inbuxa (#456): a JSON line is one object with typed fields
|
||||
#[test]
|
||||
fn json_line_is_one_typed_object() {
|
||||
let event = Event {
|
||||
inner: EventDetails {
|
||||
typ: EventType::Smtp(SmtpEvent::ConnectionStart),
|
||||
timestamp: 1_794_996_000,
|
||||
level: Level::Info,
|
||||
span: None,
|
||||
},
|
||||
keys: vec![
|
||||
(Key::RemoteIp, Value::String("192.0.2.1".into())),
|
||||
(Key::RemotePort, Value::UInt(25)),
|
||||
(
|
||||
Key::Details,
|
||||
Value::String("a \"quoted\" word\nand a line".into()),
|
||||
),
|
||||
(
|
||||
Key::Contents,
|
||||
Value::Array(vec![Value::Bool(true), Value::Int(-1)]),
|
||||
),
|
||||
],
|
||||
};
|
||||
let mut out = Vec::new();
|
||||
json_log_line(&event, &mut out).unwrap();
|
||||
|
||||
let text = String::from_utf8(out).unwrap();
|
||||
assert!(text.ends_with('\n'));
|
||||
assert_eq!(text.matches('\n').count(), 1, "one line: {text}");
|
||||
let json: serde_json::Value = serde_json::from_str(&text).unwrap();
|
||||
assert_eq!(json["timestamp"], "2026-11-18T10:00:00Z");
|
||||
assert_eq!(json["level"], "info");
|
||||
assert_eq!(json["event"], "smtp.connection-start");
|
||||
assert!(json["description"].as_str().is_some_and(|d| !d.is_empty()));
|
||||
assert_eq!(json["data"]["remoteIp"], "192.0.2.1");
|
||||
assert_eq!(json["data"]["remotePort"], 25);
|
||||
assert_eq!(json["data"]["details"], "a \"quoted\" word\nand a line");
|
||||
assert_eq!(json["data"]["contents"], serde_json::json!([true, -1]));
|
||||
}
|
||||
}
|
||||
@@ -252,6 +252,17 @@ struct LogHeader<'x> {
|
||||
}
|
||||
|
||||
fn log_header(line: &str) -> Option<LogHeader<'_>> {
|
||||
// inbuxa (#456): a Log tracer writing JSON lines
|
||||
if line.starts_with('{') {
|
||||
let entry = serde_json::from_str::<JsonLogHead<'_>>(line).ok()?;
|
||||
return Some(LogHeader {
|
||||
timestamp: DateTime::parse_from_rfc3339(entry.timestamp)
|
||||
.ok()?
|
||||
.timestamp(),
|
||||
level: TracingLevel::parse(&entry.level.to_ascii_lowercase()).map(|l| l as u8),
|
||||
event: entry.event,
|
||||
});
|
||||
}
|
||||
if !line.as_bytes().first()?.is_ascii_digit() {
|
||||
return None;
|
||||
}
|
||||
@@ -409,7 +420,53 @@ pub(crate) fn read_log_entries(
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
/// inbuxa (#456): the fields of a JSON log line that a query filters on.
|
||||
#[derive(serde::Deserialize)]
|
||||
struct JsonLogHead<'x> {
|
||||
#[serde(borrow)]
|
||||
timestamp: &'x str,
|
||||
#[serde(borrow)]
|
||||
level: &'x str,
|
||||
#[serde(borrow)]
|
||||
event: &'x str,
|
||||
}
|
||||
|
||||
/// inbuxa (#456): a JSON log line, its `data` handed back as a JSON object
|
||||
/// in `details`, so a reader gets the fields exactly as they were written.
|
||||
#[derive(serde::Deserialize)]
|
||||
struct JsonLogEntry<'x> {
|
||||
#[serde(borrow)]
|
||||
timestamp: &'x str,
|
||||
#[serde(borrow)]
|
||||
level: &'x str,
|
||||
#[serde(borrow)]
|
||||
event: &'x str,
|
||||
#[serde(default)]
|
||||
data: serde_json::Map<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
fn log_from_json_line(line: &str) -> Option<Log> {
|
||||
let entry = serde_json::from_str::<JsonLogEntry<'_>>(line.trim_end()).ok()?;
|
||||
Some(Log {
|
||||
timestamp: UTCDateTime::from_timestamp(
|
||||
DateTime::parse_from_rfc3339(entry.timestamp)
|
||||
.ok()?
|
||||
.timestamp(),
|
||||
),
|
||||
level: TracingLevel::parse(&entry.level.to_ascii_lowercase()).unwrap_or(TracingLevel::Info),
|
||||
event: EventType::parse(entry.event)?,
|
||||
details: if entry.data.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
serde_json::Value::Object(entry.data).to_string()
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
fn log_from_line(line: &str) -> Option<Log> {
|
||||
if line.starts_with('{') {
|
||||
return log_from_json_line(line);
|
||||
}
|
||||
let line = strip_ansi(line);
|
||||
let (timestamp, rest) = line.split_once(' ')?;
|
||||
let timestamp = DateTime::parse_from_rfc3339(timestamp).ok()?;
|
||||
@@ -636,7 +693,7 @@ impl<R: Read + Seek> Iterator for RevLines<R> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{LogFilter, read_log_offsets};
|
||||
use super::{LogFilter, read_log_entries, read_log_offsets};
|
||||
|
||||
#[test]
|
||||
fn limit_zero_reads_no_entries() {
|
||||
@@ -702,4 +759,133 @@ mod tests {
|
||||
assert_eq!(window, 2);
|
||||
assert_eq!(after, 2);
|
||||
}
|
||||
|
||||
// inbuxa (#456): files written as JSON lines are queried and read like
|
||||
// text ones, and a mix of both (a tracer switched over) reads in order
|
||||
#[test]
|
||||
fn reads_json_lines() {
|
||||
let dir = std::env::temp_dir().join(format!("inbuxa-log-json-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
std::fs::write(
|
||||
dir.join("inbuxa.log.2026-11-18"),
|
||||
concat!(
|
||||
"2026-11-18T10:00:00Z INFO SMTP connection started (smtp.connection-start) remoteIp = 192.0.2.1\n",
|
||||
r#"{"timestamp":"2026-11-18T10:00:01Z","level":"warn","event":"store.data-corruption","description":"Data corruption","data":{"details":"a \"quoted\" word, with a comma"}}"#,
|
||||
"\n",
|
||||
r#"{"timestamp":"2026-11-18T10:00:02Z","level":"error","event":"imap.connection-end","description":"x","data":{}}"#,
|
||||
"\n",
|
||||
r#"{"timestamp":"2026-11-18T10:00:03Z","level":"debug","event":"smtp.rcpt-to","description":"y","data":{"to":["[email protected]"],"size":12}}"#,
|
||||
"\n",
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
let ids = |filter: LogFilter| read_log_offsets(&dir, &filter, 0, 10).unwrap();
|
||||
|
||||
let all = ids(LogFilter::default());
|
||||
let at_least_warn = ids(LogFilter {
|
||||
level: Some(1),
|
||||
..Default::default()
|
||||
});
|
||||
let smtp = ids(LogFilter {
|
||||
event: Some("smtp".into()),
|
||||
..Default::default()
|
||||
});
|
||||
let window = ids(LogFilter {
|
||||
from: 1_794_996_001,
|
||||
to: 1_794_996_002,
|
||||
..Default::default()
|
||||
});
|
||||
let text = ids(LogFilter {
|
||||
text: Some("comma".into()),
|
||||
..Default::default()
|
||||
});
|
||||
let entries = read_log_entries(&dir, Some(all.clone()), 10).unwrap();
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
|
||||
assert_eq!(all.len(), 4);
|
||||
assert_eq!(at_least_warn.len(), 2, "warn and error");
|
||||
assert_eq!(smtp.len(), 2, "smtp.* only");
|
||||
assert_eq!(window.len(), 2);
|
||||
assert_eq!(text.len(), 1);
|
||||
|
||||
// Newest first
|
||||
let events = entries
|
||||
.iter()
|
||||
.map(|(_, log)| log.event.as_str())
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
events,
|
||||
[
|
||||
"smtp.rcpt-to",
|
||||
"imap.connection-end",
|
||||
"store.data-corruption",
|
||||
"smtp.connection-start"
|
||||
]
|
||||
);
|
||||
let details = |i: usize| entries[i].1.details.as_str();
|
||||
let rcpt: serde_json::Value = serde_json::from_str(details(0)).unwrap();
|
||||
assert_eq!(
|
||||
rcpt,
|
||||
serde_json::json!({"to": ["[email protected]"], "size": 12})
|
||||
);
|
||||
assert_eq!(details(1), "", "no fields, no details");
|
||||
let corruption: serde_json::Value = serde_json::from_str(details(2)).unwrap();
|
||||
assert_eq!(corruption["details"], "a \"quoted\" word, with a comma");
|
||||
assert_eq!(details(3), "remoteIp = 192.0.2.1");
|
||||
assert_eq!(entries[2].1.level as u8, 1);
|
||||
}
|
||||
|
||||
// inbuxa (#456): what the Log tracer writes as JSON reads back exactly
|
||||
#[test]
|
||||
fn json_tracer_output_reads_back() {
|
||||
use trc::{Event, EventDetails, EventType, Key, Level, SmtpEvent, Value};
|
||||
|
||||
let event = Event {
|
||||
inner: EventDetails {
|
||||
typ: EventType::Smtp(SmtpEvent::RcptTo),
|
||||
timestamp: 1_794_996_000,
|
||||
level: Level::Debug,
|
||||
span: Some(std::sync::Arc::new(Event {
|
||||
inner: EventDetails {
|
||||
typ: EventType::Smtp(SmtpEvent::ConnectionStart),
|
||||
timestamp: 1_794_995_999,
|
||||
level: Level::Info,
|
||||
span: None,
|
||||
},
|
||||
keys: vec![(Key::RemoteIp, Value::String("192.0.2.1".into()))],
|
||||
})),
|
||||
},
|
||||
keys: vec![(Key::To, Value::String("a \"b\", [email protected]".into()))],
|
||||
};
|
||||
let mut line = serde_json::to_string(&trc::serializers::json::JsonLogLine(&event)).unwrap();
|
||||
line.push('\n');
|
||||
|
||||
let dir = std::env::temp_dir().join(format!("inbuxa-log-rt-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
std::fs::write(dir.join("inbuxa.log"), &line).unwrap();
|
||||
let ids = read_log_offsets(
|
||||
&dir,
|
||||
&LogFilter {
|
||||
level: Some(3),
|
||||
event: Some("smtp.rcpt-to".into()),
|
||||
..Default::default()
|
||||
},
|
||||
0,
|
||||
10,
|
||||
)
|
||||
.unwrap();
|
||||
let entries = read_log_entries(&dir, Some(ids), 10).unwrap();
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
|
||||
assert_eq!(entries.len(), 1);
|
||||
let log = &entries[0].1;
|
||||
assert_eq!(log.event, EventType::Smtp(SmtpEvent::RcptTo));
|
||||
assert_eq!(log.level as u8, 3);
|
||||
assert_eq!(log.timestamp.timestamp(), 1_794_996_000);
|
||||
let data: serde_json::Value = serde_json::from_str(&log.details).unwrap();
|
||||
assert_eq!(
|
||||
data,
|
||||
serde_json::json!({"to": "a \"b\", [email protected]", "remoteIp": "192.0.2.1"})
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -9,11 +9,11 @@
|
||||
use crate::{
|
||||
pickle::{Pickle, PickledStream},
|
||||
schema::{
|
||||
enums::{MtaRouteType, OidcOpaqueTokens},
|
||||
enums::{LogFormat, MtaRouteType, OidcOpaqueTokens},
|
||||
prelude::{ObjectImpl, ObjectInner, ObjectType},
|
||||
structs::{
|
||||
Bootstrap, Directory, DirectoryBootstrap, Domain, Enterprise, MtaRoute, MtaRouteCommon,
|
||||
MtaRouteInbuxaRelay, OidcDirectory, Tenant,
|
||||
MtaRouteInbuxaRelay, OidcDirectory, Tenant, Tracer, TracerLog,
|
||||
},
|
||||
},
|
||||
types::{EnumImpl, map::Map},
|
||||
@@ -32,6 +32,26 @@ fn pickle_without_opaque_tokens(directory: &OidcDirectory, out: &mut Vec<u8>) {
|
||||
directory.member_tenant_id.pickle(out);
|
||||
}
|
||||
|
||||
/// A tracer as it was stored before a Log tracer had a format (#456).
|
||||
fn pickle_tracer_before_format(tracer: &Tracer, out: &mut Vec<u8>) {
|
||||
match tracer {
|
||||
Tracer::Log(log) => {
|
||||
0u16.pickle(out);
|
||||
log.path.pickle(out);
|
||||
log.prefix.pickle(out);
|
||||
log.rotate.pickle(out);
|
||||
log.ansi.pickle(out);
|
||||
log.multiline.pickle(out);
|
||||
log.enable.pickle(out);
|
||||
log.level.pickle(out);
|
||||
log.lossy.pickle(out);
|
||||
log.events.pickle(out);
|
||||
log.events_policy.pickle(out);
|
||||
}
|
||||
other => other.pickle(out),
|
||||
}
|
||||
}
|
||||
|
||||
fn read(object: ObjectType, bytes: &[u8]) -> ObjectInner {
|
||||
ObjectInner::unpickle(object, &mut PickledStream::new(bytes).unwrap()).unwrap()
|
||||
}
|
||||
@@ -77,7 +97,7 @@ fn oidc_directories_saved_before_the_opaque_token_policy_keep_userinfo() {
|
||||
bootstrap.in_memory_store.pickle(&mut bytes);
|
||||
3u16.pickle(&mut bytes);
|
||||
pickle_without_opaque_tokens(&saved, &mut bytes);
|
||||
bootstrap.tracer.pickle(&mut bytes);
|
||||
pickle_tracer_before_format(&bootstrap.tracer, &mut bytes);
|
||||
bootstrap.dns_server.pickle(&mut bytes);
|
||||
assert_eq!(
|
||||
read(ObjectType::Bootstrap, &bytes),
|
||||
@@ -219,3 +239,67 @@ fn logos_saved_before_dark_mode_read_back_without_a_dark_logo() {
|
||||
ObjectInner::Enterprise(enterprise)
|
||||
);
|
||||
}
|
||||
|
||||
// #456: Log tracers saved before the format setting read back as text, in a
|
||||
// tracer of their own and in the bootstrap, where more fields follow the
|
||||
// tracer; saved now, a JSON one reads back as JSON
|
||||
#[test]
|
||||
fn log_tracers_saved_before_the_format_read_back_as_text() {
|
||||
let log = TracerLog {
|
||||
path: "/var/log/inbuxa".to_string(),
|
||||
prefix: "inbuxa.log".to_string(),
|
||||
ansi: false,
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(log.format, LogFormat::Text);
|
||||
let tracer = Tracer::Log(log.clone());
|
||||
|
||||
// x:Tracer, version 0
|
||||
let mut bytes = vec![0u8];
|
||||
pickle_tracer_before_format(&tracer, &mut bytes);
|
||||
assert_eq!(
|
||||
read(ObjectType::Tracer, &bytes),
|
||||
ObjectInner::Tracer(tracer.clone())
|
||||
);
|
||||
|
||||
// x:Bootstrap, version 2
|
||||
let bootstrap = Bootstrap {
|
||||
server_hostname: "mx.example.org".to_string(),
|
||||
tracer: tracer.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
let mut bytes = vec![2u8];
|
||||
bootstrap.server_hostname.pickle(&mut bytes);
|
||||
bootstrap.default_domain.pickle(&mut bytes);
|
||||
bootstrap.request_tls_certificate.pickle(&mut bytes);
|
||||
bootstrap.generate_dkim_keys.pickle(&mut bytes);
|
||||
bootstrap.data_store.pickle(&mut bytes);
|
||||
bootstrap.blob_store.pickle(&mut bytes);
|
||||
bootstrap.search_store.pickle(&mut bytes);
|
||||
bootstrap.in_memory_store.pickle(&mut bytes);
|
||||
bootstrap.directory.pickle(&mut bytes);
|
||||
pickle_tracer_before_format(&bootstrap.tracer, &mut bytes);
|
||||
bootstrap.dns_server.pickle(&mut bytes);
|
||||
assert_eq!(
|
||||
read(ObjectType::Bootstrap, &bytes),
|
||||
ObjectInner::Bootstrap(bootstrap.clone())
|
||||
);
|
||||
|
||||
// Saved now
|
||||
let tracer = Tracer::Log(TracerLog {
|
||||
format: LogFormat::Json,
|
||||
..log
|
||||
});
|
||||
let bootstrap = Bootstrap {
|
||||
tracer: tracer.clone(),
|
||||
..bootstrap
|
||||
};
|
||||
assert_eq!(
|
||||
read(ObjectType::Tracer, &tracer.to_pickled_vec()),
|
||||
ObjectInner::Tracer(tracer)
|
||||
);
|
||||
assert_eq!(
|
||||
read(ObjectType::Bootstrap, &bootstrap.to_pickled_vec()),
|
||||
ObjectInner::Bootstrap(bootstrap)
|
||||
);
|
||||
}
|
||||
@@ -1272,6 +1272,15 @@ pub enum LogRotateFrequency {
|
||||
Never = 3,
|
||||
}
|
||||
|
||||
// inbuxa: #456, how a Log tracer writes each entry
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
|
||||
#[repr(u16)]
|
||||
pub enum LogFormat {
|
||||
#[default]
|
||||
Text = 0,
|
||||
Json = 1,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
|
||||
#[repr(u16)]
|
||||
pub enum LookupStoreType {
|
||||
|
||||
@@ -5522,6 +5522,57 @@ impl<'de> serde::Deserialize<'de> for Locale {
|
||||
}
|
||||
}
|
||||
|
||||
// inbuxa: #456
|
||||
impl EnumImpl for LogFormat {
|
||||
fn parse(value: &str) -> Option<Self> {
|
||||
hashify::tiny_map! {
|
||||
value.as_bytes(),
|
||||
b"text" => LogFormat::Text,
|
||||
b"json" => LogFormat::Json,
|
||||
}
|
||||
}
|
||||
|
||||
fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
LogFormat::Text => "text",
|
||||
LogFormat::Json => "json",
|
||||
}
|
||||
}
|
||||
|
||||
fn to_id(&self) -> u16 {
|
||||
*self as u16
|
||||
}
|
||||
|
||||
fn from_id(id: u16) -> Option<Self> {
|
||||
match id {
|
||||
0 => Some(LogFormat::Text),
|
||||
1 => Some(LogFormat::Json),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
const COUNT: usize = 2;
|
||||
}
|
||||
|
||||
impl serde::Serialize for LogFormat {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
serializer.serialize_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> serde::Deserialize<'de> for LogFormat {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let s = Cow::<str>::deserialize(deserializer)?;
|
||||
Self::parse(&s).ok_or_else(|| serde::de::Error::unknown_variant(&s, &[]))
|
||||
}
|
||||
}
|
||||
|
||||
impl EnumImpl for LogRotateFrequency {
|
||||
fn parse(value: &str) -> Option<Self> {
|
||||
hashify::tiny_map! {
|
||||
|
||||
@@ -6164,6 +6164,9 @@ pub struct TracerLog {
|
||||
pub events: Map<trc::EventType>,
|
||||
#[serde(rename = "eventsPolicy")]
|
||||
pub events_policy: EventPolicy,
|
||||
// inbuxa: #456
|
||||
#[serde(rename = "format")]
|
||||
pub format: LogFormat,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
|
||||
@@ -4070,8 +4070,8 @@ impl RegistryJsonPropertyPatch for BlockedIp {
|
||||
impl ObjectImpl for Bootstrap {
|
||||
const FLAGS: u64 = OBJ_SINGLETON;
|
||||
// inbuxa: 2, as x:Directory: OidcDirectory gained fields at version 2
|
||||
// of both
|
||||
const VERSION: u8 = 2;
|
||||
// of both. 3, as x:Tracer: a Log tracer's format (#456)
|
||||
const VERSION: u8 = 3;
|
||||
const OBJECT: ObjectType = ObjectType::Bootstrap;
|
||||
|
||||
fn validate(&self, errors: &mut Vec<ValidationError>) -> bool {
|
||||
@@ -46428,7 +46428,9 @@ impl RegistryJsonPropertyPatch for TraceValueUnsignedInt {
|
||||
|
||||
impl ObjectImpl for Tracer {
|
||||
const FLAGS: u64 = 0;
|
||||
const VERSION: u8 = 0;
|
||||
// inbuxa: 3, as x:Bootstrap, which holds a tracer too: a Log tracer's
|
||||
// format (#456) is stored from version 3 of both
|
||||
const VERSION: u8 = 3;
|
||||
const OBJECT: ObjectType = ObjectType::Tracer;
|
||||
|
||||
fn validate(&self, errors: &mut Vec<ValidationError>) -> bool {
|
||||
@@ -46665,6 +46667,7 @@ impl Pickle for TracerLog {
|
||||
self.lossy.pickle(out);
|
||||
self.events.pickle(out);
|
||||
self.events_policy.pickle(out);
|
||||
self.format.pickle(out);
|
||||
}
|
||||
|
||||
fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option<Self> {
|
||||
@@ -46679,6 +46682,10 @@ impl Pickle for TracerLog {
|
||||
this.lossy = Pickle::unpickle(stream)?;
|
||||
this.events = Pickle::unpickle(stream)?;
|
||||
this.events_policy = Pickle::unpickle(stream)?;
|
||||
// inbuxa: #456, stored from version 3 of x:Tracer and x:Bootstrap
|
||||
if stream.version() >= 3 {
|
||||
this.format = Pickle::unpickle(stream)?;
|
||||
}
|
||||
Some(this)
|
||||
}
|
||||
}
|
||||
@@ -46696,13 +46703,14 @@ impl Default for TracerLog {
|
||||
lossy: false,
|
||||
events: Default::default(),
|
||||
events_policy: EventPolicy::Exclude,
|
||||
format: LogFormat::Text,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl IntoValue for TracerLog {
|
||||
fn into_value(self) -> JmapValue<'static> {
|
||||
let mut map = jmap_tools::Map::with_capacity(12);
|
||||
let mut map = jmap_tools::Map::with_capacity(13);
|
||||
map.insert_unchecked(Property::Path, self.path.into_value());
|
||||
map.insert_unchecked(Property::Prefix, self.prefix.into_value());
|
||||
map.insert_unchecked(Property::Rotate, self.rotate.into_value());
|
||||
@@ -46713,6 +46721,7 @@ impl IntoValue for TracerLog {
|
||||
map.insert_unchecked(Property::Lossy, self.lossy.into_value());
|
||||
map.insert_unchecked(Property::Events, self.events.into_value());
|
||||
map.insert_unchecked(Property::EventsPolicy, self.events_policy.into_value());
|
||||
map.insert_unchecked(Property::Format, self.format.into_value());
|
||||
JmapValue::Object(map)
|
||||
}
|
||||
}
|
||||
@@ -46738,6 +46747,7 @@ impl RegistryJsonPropertyPatch for TracerLog {
|
||||
Some(Property::Lossy) => self.lossy.patch(pointer, value),
|
||||
Some(Property::Events) => self.events.patch(pointer, value),
|
||||
Some(Property::EventsPolicy) => self.events_policy.patch(pointer, value),
|
||||
Some(Property::Format) => self.format.patch(pointer, value),
|
||||
Some(Property::Type) => Ok(MaybeUnpatched::Unpatched {
|
||||
property: Property::Type,
|
||||
value,
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use crate::{Error, Event, EventDetails, EventType, Key, MetricType, Value};
|
||||
@@ -123,6 +125,47 @@ impl<T: AsRef<Event<EventDetails>>> Serialize for JsonEventSerializer<T> {
|
||||
}
|
||||
}
|
||||
|
||||
/// inbuxa (#456): one entry of a Log tracer writing JSON lines:
|
||||
/// `{"timestamp", "level", "event", "description", "data"}`, where `data`
|
||||
/// holds the event's keys (and its span's) as typed values: nested events as
|
||||
/// objects, lists as arrays.
|
||||
pub struct JsonLogLine<'x>(pub &'x Event<EventDetails>);
|
||||
|
||||
impl Serialize for JsonLogLine<'_> {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
let event = self.0;
|
||||
let mut map = serializer.serialize_map(Some(5))?;
|
||||
map.serialize_entry(
|
||||
"timestamp",
|
||||
&DateTime::from_timestamp(event.inner.timestamp as i64).to_rfc3339(),
|
||||
)?;
|
||||
map.serialize_entry("level", &event.inner.level.as_str().to_ascii_lowercase())?;
|
||||
map.serialize_entry("event", event.inner.typ.as_str())?;
|
||||
map.serialize_entry("description", event.inner.typ.description())?;
|
||||
map.serialize_entry(
|
||||
"data",
|
||||
&JsonEventSerializer {
|
||||
inner: Keys {
|
||||
keys: event.keys.as_slice(),
|
||||
span_keys: event
|
||||
.inner
|
||||
.span
|
||||
.as_ref()
|
||||
.map(|s| &s.keys[..])
|
||||
.unwrap_or(&[]),
|
||||
},
|
||||
with_spans: false,
|
||||
with_description: false,
|
||||
with_id: false,
|
||||
},
|
||||
)?;
|
||||
map.end()
|
||||
}
|
||||
}
|
||||
|
||||
impl Serialize for JsonEventSerializer<Keys<'_>> {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
|
||||
@@ -252,6 +252,17 @@ unchanged.
|
||||
raised. Both list views show it between To and Size. It isn't in the
|
||||
`text` search keywords. Like every other trace field it is kept for
|
||||
`holdTracesFor`, and only the superuser set can read it.
|
||||
- **MON-43.** (#456) A Log tracer has a `format`: `text` (the default, as
|
||||
upstream) or `json`, one object per line:
|
||||
`{"timestamp", "level", "event", "description", "data"}`, where `timestamp`
|
||||
is RFC 3339, `level` is lowercase, and `data` holds the event's keys and its
|
||||
span's as typed values (nested events as objects, lists as arrays). `ansi`
|
||||
and `multiline` apply to text only. Stored at `x:Tracer` and `x:Bootstrap`
|
||||
version 3; a tracer saved before reads back as `text`. `x:Log/get` and
|
||||
`x:Log/query`, with every filter, read both formats, a mix of them too (a
|
||||
tracer switched over keeps its older text lines). A JSON entry's `details`
|
||||
is its `data` as a JSON object, `""` when it has none; a text entry's stays
|
||||
the `key = value` text.
|
||||
|
||||
### Retention and growth
|
||||
|
||||
@@ -579,7 +590,7 @@ they cover tests 10 and 12 (`tracing.rs`), 13 (`metrics.rs`) and 18
|
||||
(`alerts.rs`). Tests 1, 3, 5 to 9, 11, 14 to 17, 19 to 21, 23 and 24 pass as
|
||||
`tests/src/system/monitoring.rs`.
|
||||
|
||||
- **MON-1 to MON-42:** built, with the limits below.
|
||||
- **MON-1 to MON-43:** built, with the limits below.
|
||||
- **Test 26 (compat)** is written as `monitoring_compat`, ignored, and unrun
|
||||
until a copy of INBUXA's data is provided. It checks observed 1, reads the
|
||||
old history without an error, and purges it. It deletes history, so only
|
||||
|
||||
@@ -134,7 +134,7 @@ These live in inbuxa's own key space (`SUBSPACE_INBUXA`) unless noted.
|
||||
| Stored traces (`x:Trace`) | network (IP, EHLO), identifier (envelope addresses, account names), metadata (message-ids, results) | correspondent, holder | `x:TracingStore`; `x:EventTracingLevel` | `x:DataRetention.holdTracesFor` (unset = unbounded) | data store, or a remote PostgreSQL/MySQL/FDB | server (records name accounts) | `common/src/telemetry/tracers/store.rs` `spawn_store_tracer`; purge `purge_spans` | **collected** (first boot inserts `TracingStore::Default` although the struct default is off); 30 d |
|
||||
| Trace search index | same keywords, lowercased | as above | `x:Search.indexTelemetry`, `indexTracingFields` | purged with the traces | search store | server | `index.rs` `trace_search_document` | indexed |
|
||||
| Stored metrics | none personal (aggregates) | — | `x:MetricsStore`, `x:Metrics` | `holdMetricsFor` | data store | server | `telemetry/metrics/store.rs` | collected, 90 d |
|
||||
| Log file | network (**client IP on every in-session line**, from the span), identifier (MAIL FROM, RCPT TO, account names), metadata (message-ids). No subject key exists; bodies only in trace-level raw-I/O events | correspondent, holder, administrator | `x:TracerLog` (`level`, `events`, `path`, `rotate`) | **unbounded: rotated daily, never deleted** | log file (`/var/log/inbuxa`); read back as `x:Log` and fed to Explain | server | `common/src/telemetry/tracers/log.rs` | **collected, info level** |
|
||||
| Log file | network (**client IP on every in-session line**, from the span), identifier (MAIL FROM, RCPT TO, account names), metadata (message-ids). No subject key exists; bodies only in trace-level raw-I/O events | correspondent, holder, administrator | `x:TracerLog` (`level`, `events`, `path`, `rotate`, `format`: text or JSON lines) | **unbounded: rotated daily, never deleted** | log file (`/var/log/inbuxa`); read back as `x:Log` and fed to Explain | server | `common/src/telemetry/tracers/log.rs` | **collected, info level** |
|
||||
| Console / journald tracers | as the log file | as above | `x:Tracer` Stdout/Journal | the host's journal policy | stdout / journald (leaves only if the host forwards it) | server | `tracers/stdout.rs`, `journald.rs` | none (recovery mode adds a console tracer) |
|
||||
| OpenTelemetry tracer | as the log file, every event and span key | as above | `x:Tracer` OtelHttp/OtelGrpc (`level`, `events`, `endpoint`) | receiver's | **external** | server | `tracers/otel.rs` | none |
|
||||
| Webhooks | as the log file; **can include raw SMTP input (message content) and model replies** — see [Findings](#findings) 1 | as above | `x:WebHook` (`events`, `eventsPolicy`, `url`) | receiver's; retried up to `discardAfter` | **external** | server | `telemetry/webhooks/mod.rs` `post_webhook_events` | none |
|
||||
|
||||
Binary file not shown.
@@ -1 +1 @@
|
||||
k2_7GSOn4Rjf0ktF3x3Z88Z1u4hUHT_I_gEoRH_viO8
|
||||
3kPB2eR8b9fyGhyDoSsRlB2AE3HhJrOzVnsdCpoURJY
|
||||
@@ -14,7 +14,7 @@ use crate::utils::{
|
||||
};
|
||||
use registry::{
|
||||
schema::{
|
||||
enums::{EventPolicy, LogRotateFrequency, TracingLevel},
|
||||
enums::{EventPolicy, LogFormat, LogRotateFrequency, TracingLevel},
|
||||
prelude::ObjectType,
|
||||
structs::{Expression, MtaStageAuth, Tracer, TracerLog},
|
||||
},
|
||||
@@ -92,6 +92,7 @@ async fn test_log_tracer_moves(test: &TestServer) {
|
||||
lossy: false,
|
||||
events: Map::new(vec![EventType::Server(ServerEvent::Licensing)]),
|
||||
events_policy: EventPolicy::Include,
|
||||
format: LogFormat::Text,
|
||||
})])
|
||||
.await;
|
||||
assert_applied(&response);
|
||||
@@ -151,6 +152,34 @@ async fn test_log_tracer_moves(test: &TestServer) {
|
||||
"no event of the stream reached the new file"
|
||||
);
|
||||
|
||||
// inbuxa (#456): switched to JSON lines, the tracer writes one object
|
||||
// per line after the text it wrote before
|
||||
let response = admin
|
||||
.registry_update(ObjectType::Tracer, [(tracer_id, json!({"format": "json"}))])
|
||||
.await;
|
||||
assert_applied(&response);
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
emit("marker-json \"quoted\"");
|
||||
wait_for(&new_file, "marker-json").await;
|
||||
let text = read(&new_file);
|
||||
let line = text
|
||||
.lines()
|
||||
.find(|line| line.contains("marker-json"))
|
||||
.unwrap();
|
||||
let entry: serde_json::Value = serde_json::from_str(line).unwrap_or_else(|err| {
|
||||
panic!("not a JSON line ({err}): {line}");
|
||||
});
|
||||
assert_eq!(entry["event"], "server.licensing", "{entry}");
|
||||
assert_eq!(
|
||||
entry["data"]["details"], "marker-json \"quoted\"",
|
||||
"{entry}"
|
||||
);
|
||||
assert!(
|
||||
text.lines()
|
||||
.any(|line| line.contains("marker-after") && !line.starts_with('{')),
|
||||
"the text written before stays text"
|
||||
);
|
||||
|
||||
// Removing the tracer stops it
|
||||
let response = admin
|
||||
.registry_destroy(ObjectType::Tracer, [tracer_id])
|
||||
|
||||
Reference in new issue
Block a user