Import upstream v0.16.22, stripped
Upstream commit: 474dd0229cb20cf513036619781ed97bd8073c3f Enterprise-only files removed or emptied: 63 Enterprise-only snippets removed: 117 in 50 files Dangling module declarations removed: 5 Cargo edits turning enterprise off: 14 Verification: clean Enterprise feature gates left for rebuilt features: 19 in 18 files Produced by tools/fork/strip.py. The full report is in docs/fork/strip-reports/ on main.
This commit is contained in:
@@ -0,0 +1,10 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
pub mod otel;
|
||||
pub mod prometheus;
|
||||
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use crate::config::telemetry::OtelMetrics;
|
||||
use opentelemetry_sdk::metrics::{
|
||||
Temporality,
|
||||
data::{
|
||||
AggregatedMetrics, Gauge, GaugeDataPoint, Histogram, HistogramDataPoint, Metric,
|
||||
MetricData, ResourceMetrics, ScopeMetrics, Sum, SumDataPoint,
|
||||
},
|
||||
exporter::PushMetricExporter,
|
||||
};
|
||||
use std::time::SystemTime;
|
||||
use trc::{Collector, TelemetryEvent};
|
||||
|
||||
impl OtelMetrics {
|
||||
pub async fn push_metrics(&self, is_enterprise: bool, start_time: SystemTime) {
|
||||
let mut metrics = Vec::with_capacity(256);
|
||||
let time = SystemTime::now();
|
||||
|
||||
// Add counters
|
||||
for counter in Collector::collect_counters(is_enterprise) {
|
||||
metrics.push(Metric::new(
|
||||
counter.id().as_str(),
|
||||
counter.id().description(),
|
||||
"events",
|
||||
AggregatedMetrics::U64(MetricData::Sum(Sum::new(
|
||||
vec![SumDataPoint::new(vec![], counter.value(), vec![])],
|
||||
start_time,
|
||||
time,
|
||||
Temporality::Cumulative,
|
||||
true,
|
||||
))),
|
||||
));
|
||||
}
|
||||
|
||||
// Add gauges
|
||||
for gauge in Collector::collect_gauges(is_enterprise) {
|
||||
metrics.push(Metric::new(
|
||||
gauge.id().as_str(),
|
||||
gauge.id().description(),
|
||||
gauge.id().unit(),
|
||||
AggregatedMetrics::U64(MetricData::Gauge(Gauge::new(
|
||||
vec![GaugeDataPoint::new(vec![], gauge.get(), vec![])],
|
||||
Some(start_time),
|
||||
time,
|
||||
))),
|
||||
));
|
||||
}
|
||||
|
||||
// Add histograms
|
||||
for histogram in Collector::collect_histograms(is_enterprise) {
|
||||
metrics.push(Metric::new(
|
||||
histogram.id().as_str(),
|
||||
histogram.id().description(),
|
||||
histogram.id().unit(),
|
||||
AggregatedMetrics::U64(MetricData::Histogram(Histogram::new(
|
||||
vec![HistogramDataPoint::new(
|
||||
vec![],
|
||||
histogram.count(),
|
||||
histogram.upper_bounds_vec(),
|
||||
histogram.buckets_vec(),
|
||||
histogram.min(),
|
||||
histogram.max(),
|
||||
histogram.sum(),
|
||||
vec![],
|
||||
)],
|
||||
start_time,
|
||||
time,
|
||||
Temporality::Cumulative,
|
||||
))),
|
||||
));
|
||||
}
|
||||
|
||||
// Export metrics
|
||||
let rm = ResourceMetrics::new(
|
||||
self.resource.clone(),
|
||||
vec![ScopeMetrics::new(self.instrumentation.clone(), metrics)],
|
||||
);
|
||||
if let Err(err) = self.exporter.export(&rm).await {
|
||||
trc::event!(
|
||||
Telemetry(TelemetryEvent::OtelMetricsExporterError),
|
||||
Reason = err.to_string(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn enable_errors() {
|
||||
// TODO: Remove this when the OpenTelemetry SDK supports error handling
|
||||
/*let _ = set_error_handler(|error| {
|
||||
trc::event!(
|
||||
Telemetry(TelemetryEvent::OtelMetricsExporterError),
|
||||
Reason = error.to_string(),
|
||||
);
|
||||
});*/
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use prometheus::{
|
||||
TextEncoder,
|
||||
proto::{Bucket, Counter, Gauge, Histogram, Metric, MetricFamily, MetricType},
|
||||
};
|
||||
use trc::{Collector, atomics::histogram::AtomicHistogram};
|
||||
|
||||
use crate::Server;
|
||||
|
||||
impl Server {
|
||||
pub async fn export_prometheus_metrics(&self) -> trc::Result<String> {
|
||||
let mut metrics = Vec::new();
|
||||
|
||||
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
let is_enterprise = false;
|
||||
|
||||
// Add counters
|
||||
for counter in Collector::collect_counters(is_enterprise) {
|
||||
let mut metric = MetricFamily::default();
|
||||
metric.set_name(metric_name(counter.id().as_str()));
|
||||
metric.set_help(counter.id().description().into());
|
||||
metric.set_field_type(MetricType::COUNTER);
|
||||
metric.set_metric(vec![new_counter(counter.value())]);
|
||||
metrics.push(metric);
|
||||
}
|
||||
|
||||
// Add gauges
|
||||
for gauge in Collector::collect_gauges(is_enterprise) {
|
||||
let mut metric = MetricFamily::default();
|
||||
metric.set_name(metric_name(gauge.id().as_str()));
|
||||
metric.set_help(gauge.id().description().into());
|
||||
metric.set_field_type(MetricType::GAUGE);
|
||||
metric.set_metric(vec![new_gauge(gauge.get())]);
|
||||
metrics.push(metric);
|
||||
}
|
||||
|
||||
// Add histograms
|
||||
for histogram in Collector::collect_histograms(is_enterprise) {
|
||||
let mut metric = MetricFamily::default();
|
||||
metric.set_name(metric_name(histogram.id().as_str()));
|
||||
metric.set_help(histogram.id().description().into());
|
||||
metric.set_field_type(MetricType::HISTOGRAM);
|
||||
metric.set_metric(vec![new_histogram(histogram)]);
|
||||
metrics.push(metric);
|
||||
}
|
||||
|
||||
TextEncoder::new().encode_to_string(&metrics).map_err(|e| {
|
||||
trc::EventType::Telemetry(trc::TelemetryEvent::OtelExporterError).reason(e)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn metric_name(id: impl AsRef<str>) -> String {
|
||||
let id = id.as_ref();
|
||||
let mut name = String::with_capacity(id.len());
|
||||
for c in id.chars() {
|
||||
if c.is_ascii_alphanumeric() {
|
||||
name.push(c);
|
||||
} else {
|
||||
name.push('_');
|
||||
}
|
||||
}
|
||||
name
|
||||
}
|
||||
|
||||
fn new_counter(value: u64) -> Metric {
|
||||
let mut m = Metric::default();
|
||||
let mut counter = Counter::default();
|
||||
counter.set_value(value as f64);
|
||||
m.set_counter(counter);
|
||||
m
|
||||
}
|
||||
|
||||
fn new_gauge(value: u64) -> Metric {
|
||||
let mut m = Metric::default();
|
||||
let mut gauge = Gauge::default();
|
||||
gauge.set_value(value as f64);
|
||||
m.set_gauge(gauge);
|
||||
m
|
||||
}
|
||||
|
||||
fn new_histogram(histogram: &AtomicHistogram<12>) -> Metric {
|
||||
let mut m = Metric::default();
|
||||
let mut h = Histogram::default();
|
||||
h.set_sample_count(histogram.count());
|
||||
h.set_sample_sum(histogram.sum() as f64);
|
||||
h.set_bucket(
|
||||
histogram
|
||||
.buckets_iter()
|
||||
.into_iter()
|
||||
.zip(histogram.upper_bounds_iter())
|
||||
.map(|(count, upper_bound)| {
|
||||
let mut b = Bucket::default();
|
||||
b.set_cumulative_count(count);
|
||||
b.set_upper_bound(if upper_bound != u64::MAX {
|
||||
upper_bound as f64
|
||||
} else {
|
||||
f64::INFINITY
|
||||
});
|
||||
b
|
||||
})
|
||||
.collect(),
|
||||
);
|
||||
m.set_histogram(h);
|
||||
m
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
pub mod metrics;
|
||||
pub mod tracers;
|
||||
pub mod webhooks;
|
||||
|
||||
use tracers::log::spawn_log_tracer;
|
||||
use tracers::otel::spawn_otel_tracer;
|
||||
use tracers::stdout::spawn_console_tracer;
|
||||
use trc::{Collector, ipc::subscriber::SubscriberBuilder};
|
||||
use webhooks::spawn_webhook_tracer;
|
||||
|
||||
use crate::config::telemetry::{Telemetry, TelemetrySubscriberType};
|
||||
|
||||
impl Telemetry {
|
||||
pub fn enable(self, is_enterprise: bool) {
|
||||
// Spawn tracers
|
||||
for tracer in self.tracers.subscribers {
|
||||
tracer.typ.spawn(
|
||||
SubscriberBuilder::new(tracer.id)
|
||||
.with_interests(tracer.interests)
|
||||
.with_lossy(tracer.lossy),
|
||||
is_enterprise,
|
||||
);
|
||||
}
|
||||
|
||||
// Update global collector
|
||||
Collector::set_interests(self.tracers.interests);
|
||||
Collector::update_custom_levels(self.tracers.levels);
|
||||
Collector::set_metrics(self.metrics);
|
||||
Collector::reload();
|
||||
}
|
||||
|
||||
pub fn update(self, is_enterprise: bool) {
|
||||
// Remove tracers that are no longer active
|
||||
let active_subscribers = Collector::get_subscribers();
|
||||
for subscribed_id in &active_subscribers {
|
||||
if !self
|
||||
.tracers
|
||||
.subscribers
|
||||
.iter()
|
||||
.any(|tracer| tracer.id == *subscribed_id)
|
||||
{
|
||||
Collector::remove_subscriber(subscribed_id.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// Activate new tracers or update existing ones
|
||||
for tracer in self.tracers.subscribers {
|
||||
if active_subscribers.contains(&tracer.id) {
|
||||
Collector::update_subscriber(tracer.id, tracer.interests, tracer.lossy);
|
||||
} else {
|
||||
tracer.typ.spawn(
|
||||
SubscriberBuilder::new(tracer.id)
|
||||
.with_interests(tracer.interests)
|
||||
.with_lossy(tracer.lossy),
|
||||
is_enterprise,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Update global collector
|
||||
Collector::set_interests(self.tracers.interests);
|
||||
Collector::update_custom_levels(self.tracers.levels);
|
||||
Collector::set_metrics(self.metrics);
|
||||
Collector::reload();
|
||||
}
|
||||
|
||||
#[cfg(feature = "test_mode")]
|
||||
pub fn test_tracer(level: trc::Level) {
|
||||
let mut interests = trc::ipc::subscriber::Interests::default();
|
||||
for event in trc::EventType::variants() {
|
||||
if level.is_contained(event.level()) {
|
||||
interests.set(*event);
|
||||
}
|
||||
}
|
||||
|
||||
spawn_console_tracer(
|
||||
SubscriberBuilder::new("stderr".to_string())
|
||||
.with_interests(interests.clone())
|
||||
.with_lossy(false),
|
||||
crate::config::telemetry::ConsoleTracer {
|
||||
ansi: true,
|
||||
multiline: false,
|
||||
buffered: false,
|
||||
},
|
||||
);
|
||||
|
||||
Collector::union_interests(interests);
|
||||
Collector::reload();
|
||||
}
|
||||
}
|
||||
|
||||
impl TelemetrySubscriberType {
|
||||
pub fn spawn(self, builder: SubscriberBuilder, is_enterprise: bool) {
|
||||
match self {
|
||||
TelemetrySubscriberType::ConsoleTracer(settings) => {
|
||||
spawn_console_tracer(builder, settings)
|
||||
}
|
||||
TelemetrySubscriberType::LogTracer(settings) => spawn_log_tracer(builder, settings),
|
||||
TelemetrySubscriberType::Webhook(settings) => spawn_webhook_tracer(builder, settings),
|
||||
TelemetrySubscriberType::OtelTracer(settings) => spawn_otel_tracer(builder, settings),
|
||||
#[cfg(unix)]
|
||||
TelemetrySubscriberType::JournalTracer(subscriber) => {
|
||||
tracers::journald::spawn_journald_tracer(builder, subscriber)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,468 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use ahash::AHashSet;
|
||||
use std::io::Write;
|
||||
use trc::ipc::subscriber::SubscriberBuilder;
|
||||
use trc::{Event, EventDetails, Level, TelemetryEvent};
|
||||
|
||||
pub(crate) fn spawn_journald_tracer(builder: SubscriberBuilder, subscriber: Subscriber) {
|
||||
let (_, mut rx) = builder.register();
|
||||
tokio::spawn(async move {
|
||||
while let Some(events) = rx.recv().await {
|
||||
for event in events {
|
||||
subscriber.send_event(&event);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
impl Subscriber {
|
||||
fn send_event(&self, event: &Event<EventDetails>) {
|
||||
let mut buf = Vec::with_capacity(256);
|
||||
put_field_wellformed(
|
||||
&mut buf,
|
||||
"PRIORITY",
|
||||
&[match event.inner.level {
|
||||
Level::Error => self.priority_mappings.error as u8,
|
||||
Level::Warn => self.priority_mappings.warn as u8,
|
||||
Level::Info => self.priority_mappings.info as u8,
|
||||
Level::Debug => self.priority_mappings.debug as u8,
|
||||
Level::Trace | Level::Disable => self.priority_mappings.trace as u8,
|
||||
}],
|
||||
);
|
||||
put_field_length_encoded(&mut buf, "SYSLOG_IDENTIFIER", |buf| {
|
||||
write!(buf, "{}", self.syslog_identifier).unwrap()
|
||||
});
|
||||
put_field_length_encoded(&mut buf, "MESSAGE", |buf| {
|
||||
write!(buf, "{}", event.inner.typ.description()).unwrap()
|
||||
});
|
||||
|
||||
let mut seen_keys = AHashSet::new();
|
||||
for (key, value) in event.keys.iter().chain(
|
||||
event
|
||||
.inner
|
||||
.span
|
||||
.as_ref()
|
||||
.map_or(([]).iter(), |span| span.keys.iter()),
|
||||
) {
|
||||
if seen_keys.insert(*key) {
|
||||
put_field_length_encoded(&mut buf, key.as_str(), |buf| {
|
||||
write!(buf, "{value}").unwrap()
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(err) = self.send_payload(&buf) {
|
||||
trc::event!(
|
||||
Telemetry(TelemetryEvent::JournalError),
|
||||
Details = "Failed to send event to journald",
|
||||
Reason = err.to_string()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SPDX-SnippetBegin
|
||||
// SPDX-FileCopyrightText: 2018 Benjamin Saunders <[email protected]>
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
use std::fs::File;
|
||||
use std::io::{self, Error, Result};
|
||||
use std::mem::{size_of, zeroed};
|
||||
#[cfg(target_os = "linux")]
|
||||
use std::os::raw::c_uint;
|
||||
use std::os::unix::ffi::OsStrExt;
|
||||
use std::os::unix::net::UnixDatagram;
|
||||
#[cfg(target_os = "linux")]
|
||||
use std::os::unix::prelude::FromRawFd;
|
||||
use std::os::unix::prelude::{AsRawFd, RawFd};
|
||||
use std::path::Path;
|
||||
use std::ptr;
|
||||
|
||||
use libc::*;
|
||||
|
||||
#[cfg(unix)]
|
||||
const JOURNALD_PATH: &str = "/run/systemd/journal/socket";
|
||||
const CMSG_BUFSIZE: usize = 64;
|
||||
|
||||
pub struct Subscriber {
|
||||
#[cfg(unix)]
|
||||
socket: UnixDatagram,
|
||||
syslog_identifier: String,
|
||||
priority_mappings: PriorityMappings,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PriorityMappings {
|
||||
/// Priority mapped to the `ERROR` level
|
||||
pub error: Priority,
|
||||
/// Priority mapped to the `WARN` level
|
||||
pub warn: Priority,
|
||||
/// Priority mapped to the `INFO` level
|
||||
pub info: Priority,
|
||||
/// Priority mapped to the `DEBUG` level
|
||||
pub debug: Priority,
|
||||
/// Priority mapped to the `TRACE` level
|
||||
pub trace: Priority,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
union AlignedBuffer<T: Copy + Clone> {
|
||||
buffer: T,
|
||||
align: cmsghdr,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
|
||||
#[repr(u8)]
|
||||
pub enum Priority {
|
||||
/// System is unusable.
|
||||
///
|
||||
/// Examples:
|
||||
///
|
||||
/// - severe Kernel BUG
|
||||
/// - systemd dumped core
|
||||
///
|
||||
/// This level should not be used by applications.
|
||||
Emergency = b'0',
|
||||
/// Should be corrected immediately.
|
||||
///
|
||||
/// Examples:
|
||||
///
|
||||
/// - Vital subsystem goes out of work, data loss:
|
||||
/// - `kernel: BUG: unable to handle kernel paging request at ffffc90403238ffc`
|
||||
Alert = b'1',
|
||||
/// Critical conditions
|
||||
///
|
||||
/// Examples:
|
||||
///
|
||||
/// - Crashe, coredumps
|
||||
/// - `systemd-coredump[25319]: Process 25310 (plugin-container) of user 1000 dumped core`
|
||||
Critical = b'2',
|
||||
/// Error conditions
|
||||
///
|
||||
/// Examples:
|
||||
///
|
||||
/// - Not severe error reported
|
||||
/// - `kernel: usb 1-3: 3:1: cannot get freq at ep 0x84, systemd[1]: Failed unmounting /var`
|
||||
/// - `libvirtd[1720]: internal error: Failed to initialize a valid firewall backend`
|
||||
Error = b'3',
|
||||
/// May indicate that an error will occur if action is not taken.
|
||||
///
|
||||
/// Examples:
|
||||
///
|
||||
/// - a non-root file system has only 1GB free
|
||||
/// - `org.freedesktop. Notifications[1860]: (process:5999): Gtk-WARNING **: Locale not supported by C library. Using the fallback 'C' locale`
|
||||
Warning = b'4',
|
||||
/// Events that are unusual, but not error conditions.
|
||||
///
|
||||
/// Examples:
|
||||
///
|
||||
/// - `systemd[1]: var.mount: Directory /var to mount over is not empty, mounting anyway`
|
||||
/// - `gcr-prompter[4997]: Gtk: GtkDialog mapped without a transient parent. This is discouraged`
|
||||
Notice = b'5',
|
||||
/// Normal operational messages that require no action.
|
||||
///
|
||||
/// Example: `lvm[585]: 7 logical volume(s) in volume group "archvg" now active`
|
||||
Informational = b'6',
|
||||
/// Information useful to developers for debugging the
|
||||
/// application.
|
||||
///
|
||||
/// Example: `kdeinit5[1900]: powerdevil: Scheduling inhibition from ":1.14" "firefox" with cookie 13 and reason "screen"`
|
||||
Debug = b'7',
|
||||
}
|
||||
|
||||
impl Subscriber {
|
||||
/// Construct a journald subscriber
|
||||
///
|
||||
/// Fails if the journald socket couldn't be opened. Returns a `NotFound` error unconditionally
|
||||
/// in non-Unix environments.
|
||||
pub fn new() -> io::Result<Self> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let socket = UnixDatagram::unbound()?;
|
||||
let sub = Self {
|
||||
socket,
|
||||
syslog_identifier: std::env::current_exe()
|
||||
.ok()
|
||||
.as_ref()
|
||||
.and_then(|p| p.file_name())
|
||||
.map(|n| n.to_string_lossy().into_owned())
|
||||
// If we fail to get the name of the current executable fall back to an empty string.
|
||||
.unwrap_or_default(),
|
||||
priority_mappings: PriorityMappings::new(),
|
||||
};
|
||||
// Check that we can talk to journald, by sending empty payload which journald discards.
|
||||
// However if the socket didn't exist or if none listened we'd get an error here.
|
||||
sub.send_payload(&[])?;
|
||||
Ok(sub)
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
Err(io::Error::new(
|
||||
io::ErrorKind::NotFound,
|
||||
"journald does not exist in this environment",
|
||||
))
|
||||
}
|
||||
|
||||
/// Sets how [`tracing_core::Level`]s are mapped to [journald priorities](Priority).
|
||||
///
|
||||
pub fn with_priority_mappings(mut self, mappings: PriorityMappings) -> Self {
|
||||
self.priority_mappings = mappings;
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the syslog identifier for this logger.
|
||||
///
|
||||
/// The syslog identifier comes from the classic syslog interface (`openlog()`
|
||||
/// and `syslog()`) and tags log entries with a given identifier.
|
||||
/// Systemd exposes it in the `SYSLOG_IDENTIFIER` journal field, and allows
|
||||
/// filtering log messages by syslog identifier with `journalctl -t`.
|
||||
/// Unlike the unit (`journalctl -u`) this field is not trusted, i.e. applications
|
||||
/// can set it freely, and use it e.g. to further categorize log entries emitted under
|
||||
/// the same systemd unit or in the same process. It also allows to filter for log
|
||||
/// entries of processes not started in their own unit.
|
||||
///
|
||||
/// See [Journal Fields](https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html)
|
||||
/// and [journalctl](https://www.freedesktop.org/software/systemd/man/journalctl.html)
|
||||
/// for more information.
|
||||
///
|
||||
/// Defaults to the file name of the executable of the current process, if any.
|
||||
pub fn with_syslog_identifier(mut self, identifier: String) -> Self {
|
||||
self.syslog_identifier = identifier;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns the syslog identifier in use.
|
||||
pub fn syslog_identifier(&self) -> &str {
|
||||
&self.syslog_identifier
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn send_payload(&self, _opayload: &[u8]) -> io::Result<()> {
|
||||
Err(io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
"journald not supported on non-Unix",
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn send_payload(&self, payload: &[u8]) -> io::Result<usize> {
|
||||
self.socket
|
||||
.send_to(payload, JOURNALD_PATH)
|
||||
.or_else(|error| {
|
||||
if Some(libc::EMSGSIZE) == error.raw_os_error() {
|
||||
self.send_large_payload(payload)
|
||||
} else {
|
||||
Err(error)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(all(unix, not(target_os = "linux")))]
|
||||
fn send_large_payload(&self, _payload: &[u8]) -> io::Result<usize> {
|
||||
Err(std::io::Error::other(
|
||||
"Large payloads not supported on non-Linux OS",
|
||||
))
|
||||
}
|
||||
|
||||
/// Send large payloads to journald via a memfd.
|
||||
#[cfg(target_os = "linux")]
|
||||
fn send_large_payload(&self, payload: &[u8]) -> io::Result<usize> {
|
||||
// If the payload's too large for a single datagram, send it through a memfd, see
|
||||
// https://systemd.io/JOURNAL_NATIVE_PROTOCOL/
|
||||
use std::os::unix::prelude::AsRawFd;
|
||||
// Write the whole payload to a memfd
|
||||
let mut mem = create_sealable()?;
|
||||
mem.write_all(payload)?;
|
||||
// Fully seal the memfd to signal journald that its backing data won't resize anymore
|
||||
// and so is safe to mmap.
|
||||
seal_fully(mem.as_raw_fd())?;
|
||||
send_one_fd_to(&self.socket, mem.as_raw_fd(), JOURNALD_PATH)
|
||||
}
|
||||
}
|
||||
|
||||
impl PriorityMappings {
|
||||
/// Returns the default priority mappings:
|
||||
///
|
||||
pub fn new() -> PriorityMappings {
|
||||
Self {
|
||||
error: Priority::Error,
|
||||
warn: Priority::Warning,
|
||||
info: Priority::Notice,
|
||||
debug: Priority::Informational,
|
||||
trace: Priority::Debug,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for PriorityMappings {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for Subscriber {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("Subscriber")
|
||||
.field("socket", &self.socket)
|
||||
.field("syslog_identifier", &self.syslog_identifier)
|
||||
.field("priority_mappings", &self.priority_mappings)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// Append a sanitized and length-encoded field into `buf`.
|
||||
///
|
||||
/// Unlike `put_field_wellformed` this function handles arbitrary field names and values.
|
||||
///
|
||||
/// `name` denotes the field name. It gets sanitized before being appended to `buf`.
|
||||
///
|
||||
/// `write_value` is invoked with `buf` as argument to append the value data to `buf`. It must
|
||||
/// not delete from `buf`, but may append arbitrary data. This function then determines the length
|
||||
/// of the data written and adds it in the appropriate place in `buf`.
|
||||
fn put_field_length_encoded(buf: &mut Vec<u8>, name: &str, write_value: impl FnOnce(&mut Vec<u8>)) {
|
||||
for ch in name.as_bytes() {
|
||||
buf.push(ch.to_ascii_uppercase());
|
||||
}
|
||||
buf.push(b'\n');
|
||||
buf.extend_from_slice(&[0; 8]); // Length tag, to be populated
|
||||
let start = buf.len();
|
||||
write_value(buf);
|
||||
let end = buf.len();
|
||||
buf[start - 8..start].copy_from_slice(&((end - start) as u64).to_le_bytes());
|
||||
buf.push(b'\n');
|
||||
}
|
||||
|
||||
/// Append arbitrary data with a well-formed name and value.
|
||||
///
|
||||
/// `value` must not contain an internal newline, because this function writes
|
||||
/// `value` in the new-line separated format.
|
||||
///
|
||||
/// For a "newline-safe" variant, see `put_field_length_encoded`.
|
||||
fn put_field_wellformed(buf: &mut Vec<u8>, name: &str, value: &[u8]) {
|
||||
buf.extend_from_slice(name.as_bytes());
|
||||
buf.push(b'\n');
|
||||
put_value(buf, value);
|
||||
}
|
||||
|
||||
/// Write the value portion of a key-value pair, in newline separated format.
|
||||
///
|
||||
/// `value` must not contain an internal newline.
|
||||
///
|
||||
/// For a "newline-safe" variant, see `put_field_length_encoded`.
|
||||
fn put_value(buf: &mut Vec<u8>, value: &[u8]) {
|
||||
buf.extend_from_slice(&(value.len() as u64).to_le_bytes());
|
||||
buf.extend_from_slice(value);
|
||||
buf.push(b'\n');
|
||||
}
|
||||
|
||||
fn assert_cmsg_bufsize() {
|
||||
let space_one_fd = unsafe { CMSG_SPACE(size_of::<RawFd>() as u32) };
|
||||
assert!(
|
||||
space_one_fd <= CMSG_BUFSIZE as u32,
|
||||
"cmsghdr buffer too small (< {}) to hold a single fd",
|
||||
space_one_fd
|
||||
);
|
||||
}
|
||||
|
||||
pub fn send_one_fd_to<P: AsRef<Path>>(socket: &UnixDatagram, fd: RawFd, path: P) -> Result<usize> {
|
||||
assert_cmsg_bufsize();
|
||||
|
||||
let mut addr: sockaddr_un = unsafe { zeroed() };
|
||||
let path_bytes = path.as_ref().as_os_str().as_bytes();
|
||||
// path_bytes may have at most sun_path + 1 bytes, to account for the trailing NUL byte.
|
||||
if addr.sun_path.len() <= path_bytes.len() {
|
||||
return Err(Error::from_raw_os_error(ENAMETOOLONG));
|
||||
}
|
||||
|
||||
addr.sun_family = AF_UNIX as _;
|
||||
unsafe {
|
||||
std::ptr::copy_nonoverlapping(
|
||||
path_bytes.as_ptr(),
|
||||
addr.sun_path.as_mut_ptr() as *mut u8,
|
||||
path_bytes.len(),
|
||||
)
|
||||
};
|
||||
|
||||
let mut msg: msghdr = unsafe { zeroed() };
|
||||
// Set the target address.
|
||||
msg.msg_name = &mut addr as *mut _ as *mut c_void;
|
||||
msg.msg_namelen = size_of::<sockaddr_un>() as socklen_t;
|
||||
|
||||
// We send no data body with this message.
|
||||
msg.msg_iov = ptr::null_mut();
|
||||
msg.msg_iovlen = 0;
|
||||
|
||||
// Create and fill the control message buffer with our file descriptor
|
||||
let mut cmsg_buffer = AlignedBuffer {
|
||||
buffer: ([0u8; CMSG_BUFSIZE]),
|
||||
};
|
||||
msg.msg_control = unsafe { cmsg_buffer.buffer.as_mut_ptr() as _ };
|
||||
msg.msg_controllen = unsafe { CMSG_SPACE(size_of::<RawFd>() as _) as _ };
|
||||
|
||||
let cmsg: &mut cmsghdr =
|
||||
unsafe { CMSG_FIRSTHDR(&msg).as_mut() }.expect("Control message buffer exhausted");
|
||||
|
||||
cmsg.cmsg_level = SOL_SOCKET;
|
||||
cmsg.cmsg_type = SCM_RIGHTS;
|
||||
cmsg.cmsg_len = unsafe { CMSG_LEN(size_of::<RawFd>() as _) as _ };
|
||||
|
||||
unsafe { ptr::write(CMSG_DATA(cmsg) as *mut RawFd, fd) };
|
||||
|
||||
let result = unsafe { sendmsg(socket.as_raw_fd(), &msg, libc::MSG_NOSIGNAL) };
|
||||
|
||||
if result < 0 {
|
||||
Err(Error::last_os_error())
|
||||
} else {
|
||||
// sendmsg returns the number of bytes written
|
||||
Ok(result as usize)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn create(flags: c_uint) -> Result<File> {
|
||||
let fd = memfd_create_syscall(flags);
|
||||
if fd < 0 {
|
||||
Err(Error::last_os_error())
|
||||
} else {
|
||||
Ok(unsafe { File::from_raw_fd(fd as RawFd) })
|
||||
}
|
||||
}
|
||||
|
||||
/// Make the `memfd_create` syscall ourself instead of going through `libc`;
|
||||
/// `memfd_create` isn't supported on `glibc<2.27` so this allows us to
|
||||
/// support old-but-still-used distros like Ubuntu Xenial, Debian Stretch,
|
||||
/// RHEL 7, etc.
|
||||
///
|
||||
/// See: https://github.com/tokio-rs/tracing/issues/1879
|
||||
#[cfg(target_os = "linux")]
|
||||
fn memfd_create_syscall(flags: c_uint) -> c_int {
|
||||
unsafe {
|
||||
syscall(
|
||||
SYS_memfd_create,
|
||||
"tracing-journald\0".as_ptr() as *const c_char,
|
||||
flags,
|
||||
) as c_int
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub fn create_sealable() -> Result<File> {
|
||||
create(MFD_ALLOW_SEALING | MFD_CLOEXEC)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub fn seal_fully(fd: RawFd) -> Result<()> {
|
||||
let all_seals = F_SEAL_SHRINK | F_SEAL_GROW | F_SEAL_WRITE | F_SEAL_SEAL;
|
||||
let result = unsafe { fcntl(fd, F_ADD_SEALS, all_seals) };
|
||||
if result < 0 {
|
||||
Err(Error::last_os_error())
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
// SPDX-SnippetEnd
|
||||
@@ -0,0 +1,141 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use std::{path::PathBuf, time::SystemTime};
|
||||
|
||||
use crate::config::telemetry::{LogTracer, RotationStrategy};
|
||||
|
||||
use mail_parser::DateTime;
|
||||
use tokio::{
|
||||
fs::{File, OpenOptions},
|
||||
io::BufWriter,
|
||||
};
|
||||
use trc::{TelemetryEvent, ipc::subscriber::SubscriberBuilder, serializers::text::FmtWriter};
|
||||
|
||||
pub(crate) fn spawn_log_tracer(builder: SubscriberBuilder, settings: LogTracer) {
|
||||
let (_, mut rx) = builder.register();
|
||||
tokio::spawn(async move {
|
||||
if let Some(writer) = settings.build_writer().await {
|
||||
let mut buf = FmtWriter::new(writer)
|
||||
.with_ansi(settings.ansi)
|
||||
.with_multiline(settings.multiline);
|
||||
let mut roatation_timestamp = settings.next_rotation();
|
||||
|
||||
while let Some(events) = rx.recv().await {
|
||||
for event in events {
|
||||
// Check if we need to rotate the log file
|
||||
if roatation_timestamp != 0 && event.inner.timestamp > roatation_timestamp {
|
||||
if let Err(err) = buf.flush().await {
|
||||
trc::event!(
|
||||
Telemetry(TelemetryEvent::LogError),
|
||||
Reason = err.to_string(),
|
||||
Details = "Failed to flush log buffer"
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(writer) = settings.build_writer().await {
|
||||
buf.update_writer(writer);
|
||||
roatation_timestamp = settings.next_rotation();
|
||||
} else {
|
||||
return;
|
||||
};
|
||||
}
|
||||
|
||||
if let Err(err) = buf.write(&event).await {
|
||||
trc::event!(
|
||||
Telemetry(TelemetryEvent::LogError),
|
||||
Reason = err.to_string(),
|
||||
Details = "Failed to write event to log"
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(err) = buf.flush().await {
|
||||
trc::event!(
|
||||
Telemetry(TelemetryEvent::LogError),
|
||||
Reason = err.to_string(),
|
||||
Details = "Failed to flush log buffer"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
impl LogTracer {
|
||||
pub async fn build_writer(&self) -> Option<BufWriter<File>> {
|
||||
let now = DateTime::from_timestamp(
|
||||
SystemTime::now()
|
||||
.duration_since(SystemTime::UNIX_EPOCH)
|
||||
.map_or(0, |d| d.as_secs()) as i64,
|
||||
);
|
||||
let file_name = match self.rotate {
|
||||
RotationStrategy::Daily => {
|
||||
format!(
|
||||
"{}.{:04}-{:02}-{:02}",
|
||||
self.prefix, now.year, now.month, now.day
|
||||
)
|
||||
}
|
||||
RotationStrategy::Hourly => {
|
||||
format!(
|
||||
"{}.{:04}-{:02}-{:02}T{:02}",
|
||||
self.prefix, now.year, now.month, now.day, now.hour
|
||||
)
|
||||
}
|
||||
RotationStrategy::Minutely => {
|
||||
format!(
|
||||
"{}.{:04}-{:02}-{:02}T{:02}:{:02}",
|
||||
self.prefix, now.year, now.month, now.day, now.hour, now.minute
|
||||
)
|
||||
}
|
||||
RotationStrategy::Never => self.prefix.clone(),
|
||||
};
|
||||
let path = PathBuf::from(&self.path).join(file_name);
|
||||
|
||||
match OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(&path)
|
||||
.await
|
||||
{
|
||||
Ok(writer) => Some(BufWriter::new(writer)),
|
||||
Err(err) => {
|
||||
trc::event!(
|
||||
Telemetry(TelemetryEvent::LogError),
|
||||
Details = "Failed to create log file",
|
||||
Path = path.to_string_lossy().into_owned(),
|
||||
Reason = err.to_string(),
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn next_rotation(&self) -> u64 {
|
||||
let mut now = DateTime::from_timestamp(
|
||||
SystemTime::now()
|
||||
.duration_since(SystemTime::UNIX_EPOCH)
|
||||
.map_or(0, |d| d.as_secs()) as i64,
|
||||
);
|
||||
|
||||
now.second = 0;
|
||||
|
||||
match self.rotate {
|
||||
RotationStrategy::Daily => {
|
||||
now.hour = 0;
|
||||
now.minute = 0;
|
||||
now.to_timestamp() as u64 + 86400
|
||||
}
|
||||
RotationStrategy::Hourly => {
|
||||
now.minute = 0;
|
||||
now.to_timestamp() as u64 + 3600
|
||||
}
|
||||
RotationStrategy::Minutely => now.to_timestamp() as u64 + 60,
|
||||
RotationStrategy::Never => 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
#[cfg(unix)]
|
||||
pub mod journald;
|
||||
pub mod log;
|
||||
pub mod otel;
|
||||
pub mod stdout;
|
||||
|
||||
|
||||
use registry::{
|
||||
schema::structs::{
|
||||
Trace, TraceEvent, TraceKeyValue, TraceValue, TraceValueBoolean, TraceValueDuration,
|
||||
TraceValueEvent, TraceValueFloat, TraceValueInteger, TraceValueIpAddr, TraceValueList,
|
||||
TraceValueString, TraceValueUTCDateTime, TraceValueUnsignedInt,
|
||||
},
|
||||
types::{datetime::UTCDateTime, ipaddr::IpAddr, list::List},
|
||||
};
|
||||
use trc::{Event, EventDetails, Value};
|
||||
|
||||
pub trait TraceEvents {
|
||||
fn build_trace_events<'x>(
|
||||
span_events: impl IntoIterator<Item = &'x Event<EventDetails>>,
|
||||
num_events: usize,
|
||||
) -> Vec<TraceEvent>;
|
||||
|
||||
fn from_events<'x>(
|
||||
span_events: impl IntoIterator<Item = &'x Event<EventDetails>>,
|
||||
num_events: usize,
|
||||
) -> Self;
|
||||
}
|
||||
|
||||
impl TraceEvents for Trace {
|
||||
fn build_trace_events<'x>(
|
||||
span_events: impl IntoIterator<Item = &'x Event<EventDetails>>,
|
||||
num_events: usize,
|
||||
) -> Vec<TraceEvent> {
|
||||
let mut events = Vec::with_capacity(num_events);
|
||||
|
||||
for event in span_events {
|
||||
let mut key_values = Vec::with_capacity(event.keys.len());
|
||||
for (key, value) in &event.keys {
|
||||
key_values.push(TraceKeyValue {
|
||||
key: *key,
|
||||
value: map_value(value),
|
||||
});
|
||||
}
|
||||
|
||||
events.push(TraceEvent {
|
||||
event: event.inner.typ,
|
||||
timestamp: UTCDateTime::from_timestamp(event.inner.timestamp as i64),
|
||||
key_values: key_values.into(),
|
||||
});
|
||||
}
|
||||
|
||||
events
|
||||
}
|
||||
|
||||
fn from_events<'x>(
|
||||
span_events: impl IntoIterator<Item = &'x Event<EventDetails>>,
|
||||
num_events: usize,
|
||||
) -> Self {
|
||||
Trace {
|
||||
events: Self::build_trace_events(span_events, num_events).into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn map_value(value: &Value) -> TraceValue {
|
||||
match value {
|
||||
Value::String(value) => TraceValue::String(TraceValueString {
|
||||
value: value.to_string(),
|
||||
}),
|
||||
Value::UInt(value) => TraceValue::UnsignedInt(TraceValueUnsignedInt { value: *value }),
|
||||
Value::Int(value) => TraceValue::Integer(TraceValueInteger { value: *value }),
|
||||
Value::Float(value) => TraceValue::Float(TraceValueFloat {
|
||||
value: (*value).into(),
|
||||
}),
|
||||
Value::Timestamp(value) => TraceValue::UTCDateTime(TraceValueUTCDateTime {
|
||||
value: UTCDateTime::from_timestamp(*value as i64),
|
||||
}),
|
||||
Value::Duration(value) => TraceValue::Duration(TraceValueDuration { value: *value }),
|
||||
Value::Bytes(items) => TraceValue::String(TraceValueString {
|
||||
value: String::from_utf8_lossy(items).to_string(),
|
||||
}),
|
||||
Value::Bool(value) => TraceValue::Boolean(TraceValueBoolean { value: *value }),
|
||||
Value::Ipv4(ipv4_addr) => TraceValue::IpAddr(TraceValueIpAddr {
|
||||
value: IpAddr((*ipv4_addr).into()),
|
||||
}),
|
||||
Value::Ipv6(ipv6_addr) => TraceValue::IpAddr(TraceValueIpAddr {
|
||||
value: IpAddr((*ipv6_addr).into()),
|
||||
}),
|
||||
Value::Event(event) => TraceValue::Event(TraceValueEvent {
|
||||
value: event
|
||||
.keys()
|
||||
.iter()
|
||||
.map(|(k, v)| TraceKeyValue {
|
||||
key: *k,
|
||||
value: map_value(v),
|
||||
})
|
||||
.collect(),
|
||||
event: event.event_type(),
|
||||
}),
|
||||
Value::Array(values) => TraceValue::List(TraceValueList {
|
||||
value: List::from_iter(values.iter().map(map_value)),
|
||||
}),
|
||||
Value::None => TraceValue::Null,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,288 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use crate::{LONG_1Y_SLUMBER, config::telemetry::OtelTracer};
|
||||
use ahash::{AHashMap, AHashSet};
|
||||
use mail_parser::DateTime;
|
||||
use opentelemetry::{
|
||||
InstrumentationScope, Key, KeyValue, Value,
|
||||
logs::{AnyValue, Severity},
|
||||
trace::{SpanContext, SpanKind, Status, TraceFlags, TraceState},
|
||||
};
|
||||
use opentelemetry_sdk::{
|
||||
Resource,
|
||||
logs::{LogBatch, LogExporter, SdkLogRecord},
|
||||
trace::{SpanData, SpanEvents, SpanExporter, SpanLinks},
|
||||
};
|
||||
use opentelemetry_semantic_conventions::resource::SERVICE_VERSION;
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
use trc::{Event, EventDetails, Level, TelemetryEvent, ipc::subscriber::SubscriberBuilder};
|
||||
|
||||
const MAX_EVENTS: usize = 2048;
|
||||
|
||||
pub(crate) fn spawn_otel_tracer(builder: SubscriberBuilder, mut otel: OtelTracer) {
|
||||
let (_, mut rx) = builder.register();
|
||||
tokio::spawn(async move {
|
||||
let resource = Resource::builder()
|
||||
.with_service_name("stalwart")
|
||||
.with_attribute(KeyValue::new(SERVICE_VERSION, env!("CARGO_PKG_VERSION")))
|
||||
.build();
|
||||
|
||||
let instrumentation = InstrumentationScope::builder("stalwart")
|
||||
.with_version(env!("CARGO_PKG_VERSION"))
|
||||
.build();
|
||||
|
||||
otel.log_exporter.set_resource(&resource);
|
||||
otel.span_exporter.set_resource(&resource);
|
||||
|
||||
let mut wakeup_time = LONG_1Y_SLUMBER;
|
||||
let mut next_delivery = Instant::now();
|
||||
|
||||
let mut pending_logs = Vec::new();
|
||||
let mut pending_spans = Vec::new();
|
||||
|
||||
let mut active_spans = AHashMap::new();
|
||||
|
||||
loop {
|
||||
// Wait for the next event or timeout
|
||||
let event_or_timeout = tokio::time::timeout(wakeup_time, rx.recv()).await;
|
||||
|
||||
match event_or_timeout {
|
||||
Ok(Some(events)) => {
|
||||
for event in events {
|
||||
if otel.log_exporter_enable {
|
||||
pending_logs.push(otel.build_log_record(&event));
|
||||
}
|
||||
|
||||
if otel.span_exporter_enable
|
||||
&& let Some(span) = event.inner.span.as_ref()
|
||||
{
|
||||
let span_id = span.span_id().unwrap();
|
||||
if !event.inner.typ.is_span_end() {
|
||||
let events = active_spans.entry(span_id).or_insert_with(Vec::new);
|
||||
if events.len() < MAX_EVENTS {
|
||||
events.push(event);
|
||||
}
|
||||
} else if let Some(events) = active_spans.remove(&span_id) {
|
||||
pending_spans.push(build_span_data(
|
||||
span,
|
||||
&event,
|
||||
events.iter().chain(std::iter::once(&event)),
|
||||
&instrumentation,
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(None) => {
|
||||
break;
|
||||
}
|
||||
Err(_) => (),
|
||||
}
|
||||
|
||||
// Process events
|
||||
let mut next_retry = None;
|
||||
let now = Instant::now();
|
||||
if next_delivery <= now {
|
||||
if !pending_spans.is_empty() || !pending_logs.is_empty() {
|
||||
next_delivery = now + otel.throttle;
|
||||
|
||||
if !pending_spans.is_empty()
|
||||
&& let Err(err) = otel
|
||||
.span_exporter
|
||||
.export(std::mem::take(&mut pending_spans))
|
||||
.await
|
||||
{
|
||||
trc::event!(
|
||||
Telemetry(TelemetryEvent::OtelExporterError),
|
||||
Details = "Failed to export spans",
|
||||
Reason = err.to_string()
|
||||
);
|
||||
}
|
||||
|
||||
if !pending_logs.is_empty() {
|
||||
let logs = pending_logs
|
||||
.iter()
|
||||
.map(|log| (log, &instrumentation))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
if let Err(err) = otel.log_exporter.export(LogBatch::new(&logs)).await {
|
||||
trc::event!(
|
||||
Telemetry(TelemetryEvent::OtelExporterError),
|
||||
Details = "Failed to export logs",
|
||||
Reason = err.to_string()
|
||||
);
|
||||
}
|
||||
pending_logs.clear();
|
||||
}
|
||||
}
|
||||
} else if !pending_logs.is_empty() || !pending_spans.is_empty() {
|
||||
// Retry later
|
||||
let this_retry = next_delivery - now;
|
||||
match next_retry {
|
||||
Some(next_retry) if this_retry >= next_retry => {}
|
||||
_ => {
|
||||
next_retry = Some(this_retry);
|
||||
}
|
||||
}
|
||||
}
|
||||
wakeup_time = next_retry.unwrap_or(LONG_1Y_SLUMBER);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn build_span_data<I, T>(
|
||||
start_span: &Event<EventDetails>,
|
||||
end_span: &Event<EventDetails>,
|
||||
span_events: I,
|
||||
instrumentation: &InstrumentationScope,
|
||||
) -> SpanData
|
||||
where
|
||||
I: IntoIterator<Item = T>,
|
||||
T: AsRef<Event<EventDetails>>,
|
||||
{
|
||||
let span_id = start_span.span_id().unwrap();
|
||||
|
||||
let mut events = SpanEvents::default();
|
||||
events.events = span_events
|
||||
.into_iter()
|
||||
.map(|event| {
|
||||
let event = event.as_ref();
|
||||
|
||||
opentelemetry::trace::Event::new(
|
||||
event.inner.typ.as_str(),
|
||||
UNIX_EPOCH + Duration::from_secs(event.inner.timestamp),
|
||||
event.keys.iter().filter_map(build_key_value).collect(),
|
||||
0,
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
|
||||
SpanData {
|
||||
span_context: SpanContext::new(
|
||||
(span_id as u128).into(),
|
||||
span_id.into(),
|
||||
TraceFlags::default(),
|
||||
false,
|
||||
TraceState::default(),
|
||||
),
|
||||
dropped_attributes_count: 0,
|
||||
parent_span_id: 0.into(),
|
||||
parent_span_is_remote: false,
|
||||
name: start_span.inner.typ.as_str().into(),
|
||||
start_time: UNIX_EPOCH + Duration::from_secs(start_span.inner.timestamp),
|
||||
end_time: UNIX_EPOCH + Duration::from_secs(end_span.inner.timestamp),
|
||||
attributes: start_span.keys.iter().filter_map(build_key_value).collect(),
|
||||
events,
|
||||
links: SpanLinks::default(),
|
||||
status: Status::default(),
|
||||
span_kind: SpanKind::Server,
|
||||
instrumentation_scope: instrumentation.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
impl OtelTracer {
|
||||
fn build_log_record(&self, event: &Event<EventDetails>) -> SdkLogRecord {
|
||||
use opentelemetry::logs::LogRecord;
|
||||
|
||||
let mut record = SdkLogRecord::new();
|
||||
record.set_event_name(event.inner.typ.as_str());
|
||||
record.set_severity_number(match event.inner.level {
|
||||
Level::Trace => Severity::Trace,
|
||||
Level::Debug => Severity::Debug,
|
||||
Level::Info => Severity::Info,
|
||||
Level::Warn => Severity::Warn,
|
||||
Level::Error => Severity::Error,
|
||||
Level::Disable => Severity::Error,
|
||||
});
|
||||
record.set_severity_text(event.inner.level.as_str());
|
||||
record.set_body(AnyValue::String(event.inner.typ.description().into()));
|
||||
record.set_timestamp(UNIX_EPOCH + Duration::from_secs(event.inner.timestamp));
|
||||
record.set_observed_timestamp(SystemTime::now());
|
||||
|
||||
if let Some(span_id) = event.span_id().filter(|span_id| *span_id != 0) {
|
||||
record.set_trace_context((span_id as u128).into(), span_id.into(), None);
|
||||
}
|
||||
|
||||
let mut seen_keys = AHashSet::new();
|
||||
for (k, v) in event.keys.iter().chain(
|
||||
event
|
||||
.inner
|
||||
.span
|
||||
.as_ref()
|
||||
.map_or(([]).iter(), |span| span.keys.iter()),
|
||||
) {
|
||||
if *k != trc::Key::SpanId && seen_keys.insert(*k) {
|
||||
record.add_attribute(k.as_str(), build_any_value(v));
|
||||
}
|
||||
}
|
||||
|
||||
record
|
||||
}
|
||||
}
|
||||
|
||||
fn build_key_value(key_value: &(trc::Key, trc::Value)) -> Option<KeyValue> {
|
||||
(key_value.0 != trc::Key::SpanId).then(|| {
|
||||
KeyValue::new(
|
||||
build_key(&key_value.0),
|
||||
match &key_value.1 {
|
||||
trc::Value::String(v) => Value::String(v.to_string().into()),
|
||||
trc::Value::UInt(v) => Value::I64(*v as i64),
|
||||
trc::Value::Int(v) => Value::I64(*v),
|
||||
trc::Value::Float(v) => Value::F64(*v),
|
||||
trc::Value::Timestamp(v) => {
|
||||
Value::String(DateTime::from_timestamp(*v as i64).to_rfc3339().into())
|
||||
}
|
||||
trc::Value::Duration(v) => Value::I64(*v as i64),
|
||||
trc::Value::Bytes(_) => Value::String("[binary data]".into()),
|
||||
trc::Value::Bool(v) => Value::Bool(*v),
|
||||
trc::Value::Ipv4(v) => Value::String(v.to_string().into()),
|
||||
trc::Value::Ipv6(v) => Value::String(v.to_string().into()),
|
||||
trc::Value::Event(_) => Value::String("[event data]".into()),
|
||||
trc::Value::Array(_) => Value::String("[array]".into()),
|
||||
trc::Value::None => Value::Bool(false),
|
||||
},
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn build_key(key: &trc::Key) -> Key {
|
||||
Key::from_static_str(key.as_str())
|
||||
}
|
||||
|
||||
fn build_any_value(value: &trc::Value) -> AnyValue {
|
||||
match value {
|
||||
trc::Value::String(v) => AnyValue::String(v.to_string().into()),
|
||||
trc::Value::UInt(v) => AnyValue::Int(*v as i64),
|
||||
trc::Value::Int(v) => AnyValue::Int(*v),
|
||||
trc::Value::Float(v) => AnyValue::Double(*v),
|
||||
trc::Value::Timestamp(v) => {
|
||||
AnyValue::String(DateTime::from_timestamp(*v as i64).to_rfc3339().into())
|
||||
}
|
||||
trc::Value::Duration(v) => AnyValue::Int(*v as i64),
|
||||
trc::Value::Bytes(v) => AnyValue::Bytes(Box::new(v.clone())),
|
||||
trc::Value::Bool(v) => AnyValue::Boolean(*v),
|
||||
trc::Value::Ipv4(v) => AnyValue::String(v.to_string().into()),
|
||||
trc::Value::Ipv6(v) => AnyValue::String(v.to_string().into()),
|
||||
trc::Value::Event(v) => AnyValue::Map(Box::new(
|
||||
[(
|
||||
Key::from_static_str("eventName"),
|
||||
AnyValue::String(v.event_type().as_str().into()),
|
||||
)]
|
||||
.into_iter()
|
||||
.chain(
|
||||
v.keys()
|
||||
.iter()
|
||||
.map(|(k, v)| (build_key(k), build_any_value(v))),
|
||||
)
|
||||
.collect(),
|
||||
)),
|
||||
trc::Value::Array(v) => {
|
||||
AnyValue::ListAny(Box::new(v.iter().map(build_any_value).collect()))
|
||||
}
|
||||
trc::Value::None => AnyValue::Boolean(false),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use std::{
|
||||
io::{Error, stderr},
|
||||
pin::Pin,
|
||||
task::{Context, Poll},
|
||||
};
|
||||
|
||||
use crate::config::telemetry::ConsoleTracer;
|
||||
use std::io::Write;
|
||||
use tokio::io::AsyncWrite;
|
||||
use trc::{ipc::subscriber::SubscriberBuilder, serializers::text::FmtWriter};
|
||||
|
||||
pub(crate) fn spawn_console_tracer(builder: SubscriberBuilder, settings: ConsoleTracer) {
|
||||
let (_, mut rx) = builder.register();
|
||||
tokio::spawn(async move {
|
||||
let mut buf = FmtWriter::new(StdErrWriter::default())
|
||||
.with_ansi(settings.ansi)
|
||||
.with_multiline(settings.multiline);
|
||||
|
||||
while let Some(events) = rx.recv().await {
|
||||
for event in events {
|
||||
let _ = buf.write(&event).await;
|
||||
|
||||
if !settings.buffered {
|
||||
let _ = buf.flush().await;
|
||||
}
|
||||
}
|
||||
|
||||
if settings.buffered {
|
||||
let _ = buf.flush().await;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const BUFFER_CAPACITY: usize = 4096;
|
||||
|
||||
pub struct StdErrWriter {
|
||||
buffer: Vec<u8>,
|
||||
}
|
||||
|
||||
impl AsyncWrite for StdErrWriter {
|
||||
fn poll_write(
|
||||
mut self: Pin<&mut Self>,
|
||||
_: &mut Context<'_>,
|
||||
bytes: &[u8],
|
||||
) -> Poll<Result<usize, Error>> {
|
||||
let bytes_len = bytes.len();
|
||||
let buffer_len = self.buffer.len();
|
||||
|
||||
if buffer_len + bytes_len < BUFFER_CAPACITY {
|
||||
self.buffer.extend_from_slice(bytes);
|
||||
Poll::Ready(Ok(bytes_len))
|
||||
} else if bytes_len > BUFFER_CAPACITY {
|
||||
let result = stderr()
|
||||
.write_all(&self.buffer)
|
||||
.and_then(|_| stderr().write_all(bytes));
|
||||
self.buffer.clear();
|
||||
Poll::Ready(result.map(|_| bytes_len))
|
||||
} else {
|
||||
let result = stderr().write_all(&self.buffer);
|
||||
self.buffer.clear();
|
||||
self.buffer.extend_from_slice(bytes);
|
||||
Poll::Ready(result.map(|_| bytes_len))
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_flush(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<Result<(), Error>> {
|
||||
Poll::Ready(if !self.buffer.is_empty() {
|
||||
let result = stderr().write_all(&self.buffer);
|
||||
self.buffer.clear();
|
||||
result
|
||||
} else {
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<Result<(), Error>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for StdErrWriter {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
buffer: Vec::with_capacity(BUFFER_CAPACITY),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use crate::{LONG_1Y_SLUMBER, config::telemetry::WebhookTracer};
|
||||
use aws_lc_rs::hmac;
|
||||
use base64::{Engine, engine::general_purpose::STANDARD};
|
||||
use serde::Serialize;
|
||||
use std::{
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
time::Instant,
|
||||
};
|
||||
use store::write::now;
|
||||
use tokio::sync::mpsc;
|
||||
use trc::{
|
||||
Event, EventDetails, ServerEvent, TelemetryEvent,
|
||||
ipc::subscriber::{EventBatch, SubscriberBuilder},
|
||||
serializers::json::JsonEventSerializer,
|
||||
};
|
||||
|
||||
pub(crate) fn spawn_webhook_tracer(builder: SubscriberBuilder, settings: WebhookTracer) {
|
||||
let (tx, mut rx) = builder.register();
|
||||
tokio::spawn(async move {
|
||||
let settings = Arc::new(settings);
|
||||
let mut wakeup_time = LONG_1Y_SLUMBER;
|
||||
let discard_after = settings.discard_after.as_secs();
|
||||
let mut pending_events = Vec::new();
|
||||
let mut next_delivery = Instant::now();
|
||||
let in_flight = Arc::new(AtomicBool::new(false));
|
||||
|
||||
loop {
|
||||
// Wait for the next event or timeout
|
||||
let event_or_timeout = tokio::time::timeout(wakeup_time, rx.recv()).await;
|
||||
let now = now();
|
||||
|
||||
match event_or_timeout {
|
||||
Ok(Some(events)) => {
|
||||
let mut discard_count = 0;
|
||||
for event in events {
|
||||
if now.saturating_sub(event.inner.timestamp) < discard_after {
|
||||
pending_events.push(event)
|
||||
} else {
|
||||
discard_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if discard_count > 0 {
|
||||
trc::event!(
|
||||
Telemetry(TelemetryEvent::WebhookError),
|
||||
Details = "Discarded stale events",
|
||||
Total = discard_count
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(None) => {
|
||||
break;
|
||||
}
|
||||
Err(_) => (),
|
||||
}
|
||||
|
||||
// Process events
|
||||
let mut next_retry = None;
|
||||
let now = Instant::now();
|
||||
if next_delivery <= now {
|
||||
if !pending_events.is_empty() {
|
||||
next_delivery = now + settings.throttle;
|
||||
if !in_flight.load(Ordering::Relaxed) {
|
||||
spawn_webhook_handler(
|
||||
settings.clone(),
|
||||
in_flight.clone(),
|
||||
std::mem::take(&mut pending_events),
|
||||
tx.clone(),
|
||||
);
|
||||
}
|
||||
}
|
||||
} else if !pending_events.is_empty() {
|
||||
// Retry later
|
||||
let this_retry = next_delivery - now;
|
||||
match next_retry {
|
||||
Some(next_retry) if this_retry >= next_retry => {}
|
||||
_ => {
|
||||
next_retry = Some(this_retry);
|
||||
}
|
||||
}
|
||||
}
|
||||
wakeup_time = next_retry.unwrap_or(LONG_1Y_SLUMBER);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct EventWrapper {
|
||||
events: JsonEventSerializer<Vec<Arc<Event<EventDetails>>>>,
|
||||
}
|
||||
|
||||
fn spawn_webhook_handler(
|
||||
settings: Arc<WebhookTracer>,
|
||||
in_flight: Arc<AtomicBool>,
|
||||
events: EventBatch,
|
||||
webhook_tx: mpsc::Sender<EventBatch>,
|
||||
) {
|
||||
tokio::spawn(async move {
|
||||
in_flight.store(true, Ordering::Relaxed);
|
||||
let wrapper = EventWrapper {
|
||||
events: JsonEventSerializer::new(events).with_id().with_spans(),
|
||||
};
|
||||
|
||||
if let Err(err) = post_webhook_events(&settings, &wrapper).await {
|
||||
trc::event!(Telemetry(TelemetryEvent::WebhookError), Details = err);
|
||||
|
||||
if webhook_tx.send(wrapper.events.into_inner()).await.is_err() {
|
||||
trc::event!(
|
||||
Server(ServerEvent::ThreadError),
|
||||
Details = "Failed to send failed webhook events back to main thread",
|
||||
CausedBy = trc::location!()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
in_flight.store(false, Ordering::Relaxed);
|
||||
});
|
||||
}
|
||||
|
||||
async fn post_webhook_events(
|
||||
settings: &WebhookTracer,
|
||||
events: &EventWrapper,
|
||||
) -> Result<(), String> {
|
||||
// Serialize body
|
||||
let body = serde_json::to_string(events)
|
||||
.map_err(|err| format!("Failed to serialize events: {}", err))?;
|
||||
|
||||
// Add HMAC-SHA256 signature
|
||||
let mut headers = settings.headers.clone();
|
||||
if !settings.key.is_empty() {
|
||||
let key = hmac::Key::new(hmac::HMAC_SHA256, settings.key.as_bytes());
|
||||
let tag = hmac::sign(&key, body.as_bytes());
|
||||
|
||||
headers.insert(
|
||||
"X-Signature",
|
||||
STANDARD.encode(tag.as_ref()).parse().unwrap(),
|
||||
);
|
||||
}
|
||||
|
||||
// Send request
|
||||
let response = settings
|
||||
.client
|
||||
.post(&settings.url)
|
||||
.timeout(settings.timeout)
|
||||
.headers(headers)
|
||||
.body(body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| format!("Webhook request to {} failed: {err}", settings.url))?;
|
||||
|
||||
if response.status().is_success() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!(
|
||||
"Webhook request to {} failed with code {}: {}",
|
||||
settings.url,
|
||||
response.status().as_u16(),
|
||||
response.status().canonical_reason().unwrap_or("Unknown")
|
||||
))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user