Scaffold Phase 0: agent -> Redpanda -> ingest -> ClickHouse -> api -> web
End-to-end log pipeline for Linux hosts, per /docs/architecture.md: - proto: shared gRPC contract (agent <-> ingest), Go bindings checked in - agent: Rust, musl-targeted, journald/file sourcing, RFC5424 parser, mTLS gRPC client, no required config for the common case - ingest: Go, single binary with --mode server|consumer|all; gRPC front end forwards to Redpanda unchanged, consumer normalizes and batch-writes to ClickHouse with at-least-once delivery - storage: ClickHouse schema + a plain SQL-file migration runner - api: minimal SELECT-only query endpoint, plain REST (not gRPC+gateway yet -- see api/README.md) - web: SvelteKit static SPA, one query page - transport: Redpanda compose + topic provisioning - cli: sentryctl ping stub - hack/dev-certs: throwaway CA + cert generation for local mTLS - root docker-compose.yml + docs/phase-0-runbook.md tie it together Not yet run end-to-end against real Docker/ClickHouse/Redpanda -- see the runbook's caveats section before relying on this working as-is.
This commit is contained in:
@@ -0,0 +1,34 @@
|
||||
[package]
|
||||
name = "sentry-agent"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
description = "Sentry distro-agnostic Linux log collector"
|
||||
|
||||
[[bin]]
|
||||
name = "sentry-agent"
|
||||
path = "src/main.rs"
|
||||
|
||||
[features]
|
||||
default = ["journald"]
|
||||
journald = []
|
||||
file-tail = []
|
||||
|
||||
[dependencies]
|
||||
sentry-parser = { path = "../sentry-parser" }
|
||||
|
||||
tokio = { version = "1", features = ["rt-multi-thread", "macros", "process", "io-util", "io-std", "time", "fs", "sync", "signal"] }
|
||||
tonic = { version = "0.12", features = ["tls"] }
|
||||
prost = "0.13"
|
||||
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
toml = "0.8"
|
||||
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
anyhow = "1"
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
|
||||
[build-dependencies]
|
||||
tonic-build = "0.12"
|
||||
@@ -0,0 +1,9 @@
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
tonic_build::configure()
|
||||
.build_server(false)
|
||||
.compile_protos(
|
||||
&["../../proto/sentry/logs/v1/logs.proto"],
|
||||
&["../../proto"],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
# Example sentry-agent config. Copy to /etc/sentry-agent/agent.toml, or
|
||||
# pass --config /path/to/this/file.
|
||||
#
|
||||
# Every field has a built-in default (see src/config.rs), so this file only
|
||||
# needs to contain what you're overriding. An agent with NO config file at
|
||||
# all still runs: it defaults to journald, service = "default", and expects
|
||||
# mTLS material at /etc/sentry-agent/{ca,client,client-key}.pem.
|
||||
|
||||
[agent]
|
||||
# host = "explicit-hostname-override" # defaults to /etc/hostname
|
||||
service = "my-service"
|
||||
|
||||
[source]
|
||||
kind = "journald"
|
||||
# unit = "nginx.service" # omit to tail the whole journal
|
||||
|
||||
# To tail a file instead:
|
||||
# [source]
|
||||
# kind = "file"
|
||||
# path = "/var/log/nginx/access.log"
|
||||
# from_beginning = false
|
||||
|
||||
[batch]
|
||||
max_size = 500
|
||||
flush_interval_ms = 2000
|
||||
|
||||
[ingest]
|
||||
endpoint = "https://ingest.internal:4317"
|
||||
|
||||
[tls]
|
||||
ca_cert = "/etc/sentry-agent/ca.pem"
|
||||
client_cert = "/etc/sentry-agent/client.pem"
|
||||
client_key = "/etc/sentry-agent/client-key.pem"
|
||||
@@ -0,0 +1,108 @@
|
||||
use crate::pb::LogRecord;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Buffers `LogRecord`s and signals when to flush, either because the
|
||||
/// buffer hit `max_size` (checked on every push) or because
|
||||
/// `flush_interval` elapsed since the last flush (checked by the caller via
|
||||
/// `poll_timeout` on a timer tick). Not thread-safe by design — one
|
||||
/// batcher per agent, driven from a single async task's select loop.
|
||||
pub struct Batcher {
|
||||
max_size: usize,
|
||||
flush_interval: Duration,
|
||||
buf: Vec<LogRecord>,
|
||||
last_flush: Instant,
|
||||
}
|
||||
|
||||
impl Batcher {
|
||||
pub fn new(max_size: usize, flush_interval: Duration) -> Self {
|
||||
Self {
|
||||
max_size,
|
||||
flush_interval,
|
||||
buf: Vec::with_capacity(max_size),
|
||||
last_flush: Instant::now(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Push a record. Returns the drained batch if this push filled the
|
||||
/// buffer to `max_size`.
|
||||
pub fn push(&mut self, record: LogRecord) -> Option<Vec<LogRecord>> {
|
||||
self.buf.push(record);
|
||||
if self.buf.len() >= self.max_size {
|
||||
Some(self.drain())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Call periodically (e.g. from a timer tick). Returns the drained
|
||||
/// batch if the flush interval has elapsed and there's anything
|
||||
/// buffered.
|
||||
pub fn poll_timeout(&mut self) -> Option<Vec<LogRecord>> {
|
||||
if !self.buf.is_empty() && self.last_flush.elapsed() >= self.flush_interval {
|
||||
Some(self.drain())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn drain(&mut self) -> Vec<LogRecord> {
|
||||
self.last_flush = Instant::now();
|
||||
std::mem::replace(&mut self.buf, Vec::with_capacity(self.max_size))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn rec(msg: &str) -> LogRecord {
|
||||
LogRecord {
|
||||
timestamp_unix_nano: 0,
|
||||
host: "h".into(),
|
||||
service: "s".into(),
|
||||
severity: 0,
|
||||
message: msg.into(),
|
||||
attributes: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flushes_on_size() {
|
||||
let mut b = Batcher::new(2, Duration::from_secs(999));
|
||||
assert!(b.push(rec("a")).is_none());
|
||||
let batch = b.push(rec("b")).expect("should flush at max_size");
|
||||
assert_eq!(batch.len(), 2);
|
||||
assert_eq!(batch[0].message, "a");
|
||||
assert_eq!(batch[1].message, "b");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn buffer_empty_after_size_flush() {
|
||||
let mut b = Batcher::new(1, Duration::from_secs(999));
|
||||
b.push(rec("a")).expect("flush at max_size 1");
|
||||
assert!(b.poll_timeout().is_none(), "buffer should be empty post-flush");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flushes_on_timeout() {
|
||||
let mut b = Batcher::new(100, Duration::from_millis(10));
|
||||
assert!(b.push(rec("a")).is_none());
|
||||
std::thread::sleep(Duration::from_millis(30));
|
||||
let batch = b.poll_timeout().expect("should flush after timeout");
|
||||
assert_eq!(batch.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_flush_when_buffer_empty() {
|
||||
let mut b = Batcher::new(10, Duration::from_millis(1));
|
||||
std::thread::sleep(Duration::from_millis(5));
|
||||
assert!(b.poll_timeout().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_flush_before_timeout_elapsed() {
|
||||
let mut b = Batcher::new(10, Duration::from_secs(999));
|
||||
b.push(rec("a"));
|
||||
assert!(b.poll_timeout().is_none());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
use anyhow::{Context, Result};
|
||||
use serde::Deserialize;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
const DEFAULT_CONFIG_PATH: &str = "/etc/sentry-agent/agent.toml";
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Default)]
|
||||
#[serde(default)]
|
||||
pub struct Config {
|
||||
pub agent: AgentConfig,
|
||||
pub source: SourceConfig,
|
||||
pub batch: BatchConfig,
|
||||
pub ingest: IngestConfig,
|
||||
pub tls: TlsConfig,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
/// Loads config from `explicit_path` if given, else from
|
||||
/// `/etc/sentry-agent/agent.toml` if it exists, else falls back to
|
||||
/// built-in defaults (journald source, default TLS cert paths). Only an
|
||||
/// explicitly-passed `--config` path that doesn't exist is an error;
|
||||
/// the conventional default path is optional.
|
||||
pub fn load(explicit_path: Option<&Path>) -> Result<Config> {
|
||||
let path = match explicit_path {
|
||||
Some(p) => Some(p.to_path_buf()),
|
||||
None => {
|
||||
let default = PathBuf::from(DEFAULT_CONFIG_PATH);
|
||||
default.exists().then_some(default)
|
||||
}
|
||||
};
|
||||
|
||||
match path {
|
||||
Some(p) => {
|
||||
let raw = std::fs::read_to_string(&p)
|
||||
.with_context(|| format!("reading config file {}", p.display()))?;
|
||||
toml::from_str(&raw).with_context(|| format!("parsing config file {}", p.display()))
|
||||
}
|
||||
None => Ok(Config::default()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct AgentConfig {
|
||||
/// Overrides the auto-detected system hostname. Defaults to reading
|
||||
/// /etc/hostname at startup when unset.
|
||||
pub host: Option<String>,
|
||||
pub service: String,
|
||||
}
|
||||
|
||||
impl Default for AgentConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
host: None,
|
||||
service: "default".to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "lowercase", tag = "kind")]
|
||||
pub enum SourceConfig {
|
||||
Journald {
|
||||
#[serde(default)]
|
||||
unit: Option<String>,
|
||||
},
|
||||
File {
|
||||
path: PathBuf,
|
||||
#[serde(default)]
|
||||
from_beginning: bool,
|
||||
},
|
||||
}
|
||||
|
||||
impl Default for SourceConfig {
|
||||
fn default() -> Self {
|
||||
SourceConfig::Journald { unit: None }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct BatchConfig {
|
||||
pub max_size: usize,
|
||||
pub flush_interval_ms: u64,
|
||||
}
|
||||
|
||||
impl Default for BatchConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_size: 500,
|
||||
flush_interval_ms: 2000,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct IngestConfig {
|
||||
pub endpoint: String,
|
||||
}
|
||||
|
||||
impl Default for IngestConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
endpoint: "https://127.0.0.1:4317".to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct TlsConfig {
|
||||
pub ca_cert: PathBuf,
|
||||
pub client_cert: PathBuf,
|
||||
pub client_key: PathBuf,
|
||||
}
|
||||
|
||||
impl Default for TlsConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
ca_cert: PathBuf::from("/etc/sentry-agent/ca.pem"),
|
||||
client_cert: PathBuf::from("/etc/sentry-agent/client.pem"),
|
||||
client_key: PathBuf::from("/etc/sentry-agent/client-key.pem"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
use crate::config::{IngestConfig, TlsConfig};
|
||||
use crate::pb::{log_ingest_client::LogIngestClient, LogRecord, PushBatchRequest};
|
||||
use anyhow::{Context, Result};
|
||||
use tonic::transport::{Certificate, Channel, ClientTlsConfig, Identity};
|
||||
|
||||
/// Establishes an mTLS gRPC channel to the ingest service. Agents never
|
||||
/// talk to Redpanda directly — this is the only network egress the agent
|
||||
/// has, by design (see /docs/architecture.md).
|
||||
pub async fn connect(ingest: &IngestConfig, tls: &TlsConfig) -> Result<LogIngestClient<Channel>> {
|
||||
let ca = tokio::fs::read(&tls.ca_cert)
|
||||
.await
|
||||
.with_context(|| format!("reading CA cert at {}", tls.ca_cert.display()))?;
|
||||
let cert = tokio::fs::read(&tls.client_cert)
|
||||
.await
|
||||
.with_context(|| format!("reading client cert at {}", tls.client_cert.display()))?;
|
||||
let key = tokio::fs::read(&tls.client_key)
|
||||
.await
|
||||
.with_context(|| format!("reading client key at {}", tls.client_key.display()))?;
|
||||
|
||||
let tls_config = ClientTlsConfig::new()
|
||||
.ca_certificate(Certificate::from_pem(ca))
|
||||
.identity(Identity::from_pem(cert, key));
|
||||
|
||||
let channel = Channel::from_shared(ingest.endpoint.clone())
|
||||
.context("invalid ingest endpoint URL")?
|
||||
.tls_config(tls_config)
|
||||
.context("configuring mTLS")?
|
||||
.connect()
|
||||
.await
|
||||
.context("connecting to ingest service")?;
|
||||
|
||||
Ok(LogIngestClient::new(channel))
|
||||
}
|
||||
|
||||
pub async fn send_batch(
|
||||
client: &mut LogIngestClient<Channel>,
|
||||
batch_id: String,
|
||||
records: Vec<LogRecord>,
|
||||
) -> Result<u32> {
|
||||
let resp = client
|
||||
.push_batch(PushBatchRequest { batch_id, records })
|
||||
.await
|
||||
.context("PushBatch RPC failed")?;
|
||||
Ok(resp.into_inner().accepted)
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
mod batch;
|
||||
mod config;
|
||||
mod grpc;
|
||||
mod source;
|
||||
|
||||
pub mod pb {
|
||||
tonic::include_proto!("sentry.logs.v1");
|
||||
}
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use batch::Batcher;
|
||||
use clap::Parser;
|
||||
use config::Config;
|
||||
use pb::{log_ingest_client::LogIngestClient, LogRecord, Severity};
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::mpsc;
|
||||
use tonic::transport::Channel;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "sentry-agent", about = "Sentry Linux log collector")]
|
||||
struct Cli {
|
||||
/// Path to a TOML config file. Defaults to /etc/sentry-agent/agent.toml
|
||||
/// if present, otherwise built-in defaults (journald source, default
|
||||
/// TLS cert paths under /etc/sentry-agent/).
|
||||
#[arg(long)]
|
||||
config: Option<PathBuf>,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
|
||||
.init();
|
||||
|
||||
let cli = Cli::parse();
|
||||
let cfg = Config::load(cli.config.as_deref()).context("loading config")?;
|
||||
|
||||
let host = cfg.agent.host.clone().unwrap_or_else(default_hostname);
|
||||
let service = cfg.agent.service.clone();
|
||||
|
||||
let (tx, mut rx) = mpsc::channel(1024);
|
||||
let source_handle = tokio::spawn(spawn_source(cfg.source.clone(), tx));
|
||||
|
||||
let mut client = grpc::connect(&cfg.ingest, &cfg.tls)
|
||||
.await
|
||||
.context("connecting to ingest service")?;
|
||||
tracing::info!(endpoint = %cfg.ingest.endpoint, "connected to ingest service");
|
||||
|
||||
let flush_interval = Duration::from_millis(cfg.batch.flush_interval_ms);
|
||||
let mut batcher = Batcher::new(cfg.batch.max_size, flush_interval);
|
||||
let mut ticker = tokio::time::interval(flush_interval.max(Duration::from_millis(50)));
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
maybe_line = rx.recv() => {
|
||||
let Some(raw) = maybe_line else {
|
||||
tracing::warn!("source exited, flushing remaining batch and shutting down");
|
||||
break;
|
||||
};
|
||||
let parsed = sentry_parser::parse(&raw.line);
|
||||
let severity = to_pb_severity(raw.severity_hint.or(parsed.severity));
|
||||
let record = LogRecord {
|
||||
timestamp_unix_nano: raw.timestamp_unix_nano,
|
||||
host: host.clone(),
|
||||
service: service.clone(),
|
||||
severity: severity as i32,
|
||||
message: parsed.message,
|
||||
attributes: parsed.attributes.into_iter().collect(),
|
||||
};
|
||||
if let Some(batch) = batcher.push(record) {
|
||||
flush(&mut client, batch).await;
|
||||
}
|
||||
}
|
||||
_ = ticker.tick() => {
|
||||
if let Some(batch) = batcher.poll_timeout() {
|
||||
flush(&mut client, batch).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(batch) = batcher.poll_timeout() {
|
||||
flush(&mut client, batch).await;
|
||||
}
|
||||
source_handle.abort();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn spawn_source(source: config::SourceConfig, tx: source::LineSender) {
|
||||
let result = match source {
|
||||
#[cfg(feature = "journald")]
|
||||
config::SourceConfig::Journald { unit } => source::journald::run(unit.as_deref(), tx).await,
|
||||
#[cfg(not(feature = "journald"))]
|
||||
config::SourceConfig::Journald { .. } => {
|
||||
Err(anyhow::anyhow!("this build was compiled without the `journald` feature"))
|
||||
}
|
||||
|
||||
#[cfg(feature = "file-tail")]
|
||||
config::SourceConfig::File { path, from_beginning } => {
|
||||
source::file_tail::run(&path, from_beginning, tx).await
|
||||
}
|
||||
#[cfg(not(feature = "file-tail"))]
|
||||
config::SourceConfig::File { .. } => {
|
||||
Err(anyhow::anyhow!("this build was compiled without the `file-tail` feature"))
|
||||
}
|
||||
};
|
||||
if let Err(e) = result {
|
||||
tracing::error!(error = %e, "log source exited with error");
|
||||
}
|
||||
}
|
||||
|
||||
async fn flush(client: &mut LogIngestClient<Channel>, batch: Vec<LogRecord>) {
|
||||
let n = batch.len();
|
||||
let batch_id = batch_id();
|
||||
match grpc::send_batch(client, batch_id, batch).await {
|
||||
Ok(accepted) => tracing::debug!(accepted, sent = n, "batch flushed"),
|
||||
Err(e) => tracing::error!(error = %e, sent = n, "batch flush failed"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Best-effort batch identifier for ingest-side dedup on retry. Not
|
||||
/// globally unique (host + nanosecond timestamp), which is good enough for
|
||||
/// Phase 0's single-agent-per-host reality; revisit if agents ever share
|
||||
/// an identity or clock resolution becomes a problem.
|
||||
fn batch_id() -> String {
|
||||
let nanos = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_nanos())
|
||||
.unwrap_or(0);
|
||||
format!("{nanos:x}")
|
||||
}
|
||||
|
||||
fn to_pb_severity(sev: Option<u8>) -> Severity {
|
||||
match sev {
|
||||
Some(0..=2) => Severity::Fatal, // emerg / alert / crit
|
||||
Some(3) => Severity::Error, // err
|
||||
Some(4) => Severity::Warn, // warning
|
||||
Some(5) | Some(6) => Severity::Info, // notice / info
|
||||
Some(7) => Severity::Debug, // debug
|
||||
_ => Severity::Unspecified,
|
||||
}
|
||||
}
|
||||
|
||||
fn default_hostname() -> String {
|
||||
if let Ok(s) = std::fs::read_to_string("/etc/hostname") {
|
||||
let s = s.trim().to_string();
|
||||
if !s.is_empty() {
|
||||
return s;
|
||||
}
|
||||
}
|
||||
std::env::var("HOSTNAME").unwrap_or_else(|_| "unknown-host".to_string())
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
use super::{LineSender, RawLine};
|
||||
use anyhow::{Context, Result};
|
||||
use std::io::SeekFrom;
|
||||
use std::path::Path;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
use tokio::fs::File;
|
||||
use tokio::io::{AsyncBufReadExt, AsyncSeekExt, BufReader};
|
||||
|
||||
const POLL_INTERVAL: Duration = Duration::from_millis(500);
|
||||
|
||||
/// Polling-based file tailer: no inotify/`notify` crate dependency. Good
|
||||
/// enough for Phase 0 (journald is the primary source). Handles basic
|
||||
/// truncation (e.g. logrotate `copytruncate`) by detecting the file shrank
|
||||
/// and reopening from the start. Does not follow rename-based rotation
|
||||
/// (logrotate `create`) — that's deferred until file-tail is more than a
|
||||
/// fallback path.
|
||||
pub async fn run(path: &Path, from_beginning: bool, tx: LineSender) -> Result<()> {
|
||||
let file = File::open(path)
|
||||
.await
|
||||
.with_context(|| format!("opening {}", path.display()))?;
|
||||
|
||||
let mut pos = if from_beginning { 0 } else { file.metadata().await?.len() };
|
||||
|
||||
let mut reader = BufReader::new(file);
|
||||
reader.seek(SeekFrom::Start(pos)).await?;
|
||||
let mut buf = String::new();
|
||||
|
||||
loop {
|
||||
buf.clear();
|
||||
let n = reader
|
||||
.read_line(&mut buf)
|
||||
.await
|
||||
.context("reading line from file")?;
|
||||
if n == 0 {
|
||||
let metadata = tokio::fs::metadata(path).await.context("stat-ing file")?;
|
||||
if metadata.len() < pos {
|
||||
tracing::warn!(path = %path.display(), "file shrank, assuming truncation and reopening from start");
|
||||
let f = File::open(path)
|
||||
.await
|
||||
.context("reopening file after truncation")?;
|
||||
reader = BufReader::new(f);
|
||||
pos = 0;
|
||||
}
|
||||
tokio::time::sleep(POLL_INTERVAL).await;
|
||||
continue;
|
||||
}
|
||||
pos += n as u64;
|
||||
|
||||
let line = buf.trim_end_matches(['\n', '\r']).to_string();
|
||||
if line.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let timestamp_unix_nano = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_nanos() as i64)
|
||||
.unwrap_or(0);
|
||||
|
||||
if tx
|
||||
.send(RawLine {
|
||||
line,
|
||||
timestamp_unix_nano,
|
||||
severity_hint: None,
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
use super::{LineSender, RawLine};
|
||||
use anyhow::{Context, Result};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use tokio::io::{AsyncBufReadExt, BufReader};
|
||||
use tokio::process::Command;
|
||||
|
||||
/// Reads journald entries by shelling out to `journalctl -f -o json`
|
||||
/// rather than linking libsystemd via FFI. Linking libsystemd into a
|
||||
/// statically-linked musl binary is fragile (it pulls in dbus/libcap
|
||||
/// transitively and isn't designed for static linking) and would work
|
||||
/// against the no-glibc-runtime-deps constraint in spirit even where it's
|
||||
/// technically possible. `journalctl` ships on every systemd distro this
|
||||
/// agent targets, so shelling out avoids the problem entirely. See
|
||||
/// /docs/architecture.md.
|
||||
pub async fn run(unit: Option<&str>, tx: LineSender) -> Result<()> {
|
||||
let mut cmd = Command::new("journalctl");
|
||||
cmd.arg("-f")
|
||||
.arg("-o")
|
||||
.arg("json")
|
||||
.arg("--since=now")
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::null());
|
||||
if let Some(unit) = unit {
|
||||
cmd.arg("-u").arg(unit);
|
||||
}
|
||||
|
||||
let mut child = cmd
|
||||
.spawn()
|
||||
.context("spawning journalctl -f -o json (is systemd-journal installed?)")?;
|
||||
let stdout = child.stdout.take().context("journalctl child had no stdout")?;
|
||||
let mut lines = BufReader::new(stdout).lines();
|
||||
|
||||
while let Some(line) = lines.next_line().await.context("reading journalctl output")? {
|
||||
let Ok(entry) = serde_json::from_str::<serde_json::Value>(&line) else {
|
||||
tracing::warn!(%line, "skipping unparseable journalctl JSON line");
|
||||
continue;
|
||||
};
|
||||
|
||||
let message = entry
|
||||
.get("MESSAGE")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or_default()
|
||||
.to_string();
|
||||
if message.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let severity_hint = entry
|
||||
.get("PRIORITY")
|
||||
.and_then(|v| v.as_str().map(str::to_string).or_else(|| v.as_u64().map(|n| n.to_string())))
|
||||
.and_then(|s| s.parse::<u8>().ok())
|
||||
.filter(|&p| p <= 7);
|
||||
|
||||
let timestamp_unix_nano = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_nanos() as i64)
|
||||
.unwrap_or(0);
|
||||
|
||||
if tx
|
||||
.send(RawLine {
|
||||
line: message,
|
||||
timestamp_unix_nano,
|
||||
severity_hint,
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
break; // receiver dropped, agent is shutting down
|
||||
}
|
||||
}
|
||||
|
||||
let status = child.wait().await.context("waiting for journalctl to exit")?;
|
||||
tracing::warn!(?status, "journalctl exited");
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
/// A raw line read from a source, plus whatever metadata the source itself
|
||||
/// already knows before the RFC 5424 parser ever sees it.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RawLine {
|
||||
pub line: String,
|
||||
/// Unix epoch nanoseconds at time of read.
|
||||
pub timestamp_unix_nano: i64,
|
||||
/// Syslog severity (0-7) if the source already knows it independent of
|
||||
/// the line's own content — e.g. journald's PRIORITY field. When set,
|
||||
/// this takes precedence over whatever the RFC 5424 parser infers from
|
||||
/// the message text, since it comes from a more authoritative place.
|
||||
pub severity_hint: Option<u8>,
|
||||
}
|
||||
|
||||
pub type LineSender = mpsc::Sender<RawLine>;
|
||||
|
||||
#[cfg(feature = "journald")]
|
||||
pub mod journald;
|
||||
|
||||
#[cfg(feature = "file-tail")]
|
||||
pub mod file_tail;
|
||||
Reference in New Issue
Block a user