Scaffold Phase 0: agent -> Redpanda -> ingest -> ClickHouse -> api -> web

End-to-end log pipeline for Linux hosts, per /docs/architecture.md:

- proto: shared gRPC contract (agent <-> ingest), Go bindings checked in
- agent: Rust, musl-targeted, journald/file sourcing, RFC5424 parser,
  mTLS gRPC client, no required config for the common case
- ingest: Go, single binary with --mode server|consumer|all; gRPC front
  end forwards to Redpanda unchanged, consumer normalizes and
  batch-writes to ClickHouse with at-least-once delivery
- storage: ClickHouse schema + a plain SQL-file migration runner
- api: minimal SELECT-only query endpoint, plain REST (not gRPC+gateway
  yet -- see api/README.md)
- web: SvelteKit static SPA, one query page
- transport: Redpanda compose + topic provisioning
- cli: sentryctl ping stub
- hack/dev-certs: throwaway CA + cert generation for local mTLS
- root docker-compose.yml + docs/phase-0-runbook.md tie it together

Not yet run end-to-end against real Docker/ClickHouse/Redpanda -- see the
runbook's caveats section before relying on this working as-is.
This commit is contained in:
2026-08-13 08:25:19 -07:00
commit b6b092c912
92 changed files with 7796 additions and 0 deletions
+34
View File
@@ -0,0 +1,34 @@
[package]
name = "sentry-agent"
version.workspace = true
edition.workspace = true
license.workspace = true
description = "Sentry distro-agnostic Linux log collector"
[[bin]]
name = "sentry-agent"
path = "src/main.rs"
[features]
default = ["journald"]
journald = []
file-tail = []
[dependencies]
sentry-parser = { path = "../sentry-parser" }
tokio = { version = "1", features = ["rt-multi-thread", "macros", "process", "io-util", "io-std", "time", "fs", "sync", "signal"] }
tonic = { version = "0.12", features = ["tls"] }
prost = "0.13"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
toml = "0.8"
clap = { version = "4", features = ["derive"] }
anyhow = "1"
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
[build-dependencies]
tonic-build = "0.12"
+9
View File
@@ -0,0 +1,9 @@
fn main() -> Result<(), Box<dyn std::error::Error>> {
tonic_build::configure()
.build_server(false)
.compile_protos(
&["../../proto/sentry/logs/v1/logs.proto"],
&["../../proto"],
)?;
Ok(())
}
@@ -0,0 +1,33 @@
# Example sentry-agent config. Copy to /etc/sentry-agent/agent.toml, or
# pass --config /path/to/this/file.
#
# Every field has a built-in default (see src/config.rs), so this file only
# needs to contain what you're overriding. An agent with NO config file at
# all still runs: it defaults to journald, service = "default", and expects
# mTLS material at /etc/sentry-agent/{ca,client,client-key}.pem.
[agent]
# host = "explicit-hostname-override" # defaults to /etc/hostname
service = "my-service"
[source]
kind = "journald"
# unit = "nginx.service" # omit to tail the whole journal
# To tail a file instead:
# [source]
# kind = "file"
# path = "/var/log/nginx/access.log"
# from_beginning = false
[batch]
max_size = 500
flush_interval_ms = 2000
[ingest]
endpoint = "https://ingest.internal:4317"
[tls]
ca_cert = "/etc/sentry-agent/ca.pem"
client_cert = "/etc/sentry-agent/client.pem"
client_key = "/etc/sentry-agent/client-key.pem"
+108
View File
@@ -0,0 +1,108 @@
use crate::pb::LogRecord;
use std::time::{Duration, Instant};
/// Buffers `LogRecord`s and signals when to flush, either because the
/// buffer hit `max_size` (checked on every push) or because
/// `flush_interval` elapsed since the last flush (checked by the caller via
/// `poll_timeout` on a timer tick). Not thread-safe by design — one
/// batcher per agent, driven from a single async task's select loop.
pub struct Batcher {
max_size: usize,
flush_interval: Duration,
buf: Vec<LogRecord>,
last_flush: Instant,
}
impl Batcher {
pub fn new(max_size: usize, flush_interval: Duration) -> Self {
Self {
max_size,
flush_interval,
buf: Vec::with_capacity(max_size),
last_flush: Instant::now(),
}
}
/// Push a record. Returns the drained batch if this push filled the
/// buffer to `max_size`.
pub fn push(&mut self, record: LogRecord) -> Option<Vec<LogRecord>> {
self.buf.push(record);
if self.buf.len() >= self.max_size {
Some(self.drain())
} else {
None
}
}
/// Call periodically (e.g. from a timer tick). Returns the drained
/// batch if the flush interval has elapsed and there's anything
/// buffered.
pub fn poll_timeout(&mut self) -> Option<Vec<LogRecord>> {
if !self.buf.is_empty() && self.last_flush.elapsed() >= self.flush_interval {
Some(self.drain())
} else {
None
}
}
fn drain(&mut self) -> Vec<LogRecord> {
self.last_flush = Instant::now();
std::mem::replace(&mut self.buf, Vec::with_capacity(self.max_size))
}
}
#[cfg(test)]
mod tests {
use super::*;
fn rec(msg: &str) -> LogRecord {
LogRecord {
timestamp_unix_nano: 0,
host: "h".into(),
service: "s".into(),
severity: 0,
message: msg.into(),
attributes: Default::default(),
}
}
#[test]
fn flushes_on_size() {
let mut b = Batcher::new(2, Duration::from_secs(999));
assert!(b.push(rec("a")).is_none());
let batch = b.push(rec("b")).expect("should flush at max_size");
assert_eq!(batch.len(), 2);
assert_eq!(batch[0].message, "a");
assert_eq!(batch[1].message, "b");
}
#[test]
fn buffer_empty_after_size_flush() {
let mut b = Batcher::new(1, Duration::from_secs(999));
b.push(rec("a")).expect("flush at max_size 1");
assert!(b.poll_timeout().is_none(), "buffer should be empty post-flush");
}
#[test]
fn flushes_on_timeout() {
let mut b = Batcher::new(100, Duration::from_millis(10));
assert!(b.push(rec("a")).is_none());
std::thread::sleep(Duration::from_millis(30));
let batch = b.poll_timeout().expect("should flush after timeout");
assert_eq!(batch.len(), 1);
}
#[test]
fn no_flush_when_buffer_empty() {
let mut b = Batcher::new(10, Duration::from_millis(1));
std::thread::sleep(Duration::from_millis(5));
assert!(b.poll_timeout().is_none());
}
#[test]
fn no_flush_before_timeout_elapsed() {
let mut b = Batcher::new(10, Duration::from_secs(999));
b.push(rec("a"));
assert!(b.poll_timeout().is_none());
}
}
+127
View File
@@ -0,0 +1,127 @@
use anyhow::{Context, Result};
use serde::Deserialize;
use std::path::{Path, PathBuf};
const DEFAULT_CONFIG_PATH: &str = "/etc/sentry-agent/agent.toml";
#[derive(Debug, Clone, Deserialize, Default)]
#[serde(default)]
pub struct Config {
pub agent: AgentConfig,
pub source: SourceConfig,
pub batch: BatchConfig,
pub ingest: IngestConfig,
pub tls: TlsConfig,
}
impl Config {
/// Loads config from `explicit_path` if given, else from
/// `/etc/sentry-agent/agent.toml` if it exists, else falls back to
/// built-in defaults (journald source, default TLS cert paths). Only an
/// explicitly-passed `--config` path that doesn't exist is an error;
/// the conventional default path is optional.
pub fn load(explicit_path: Option<&Path>) -> Result<Config> {
let path = match explicit_path {
Some(p) => Some(p.to_path_buf()),
None => {
let default = PathBuf::from(DEFAULT_CONFIG_PATH);
default.exists().then_some(default)
}
};
match path {
Some(p) => {
let raw = std::fs::read_to_string(&p)
.with_context(|| format!("reading config file {}", p.display()))?;
toml::from_str(&raw).with_context(|| format!("parsing config file {}", p.display()))
}
None => Ok(Config::default()),
}
}
}
#[derive(Debug, Clone, Deserialize)]
#[serde(default)]
pub struct AgentConfig {
/// Overrides the auto-detected system hostname. Defaults to reading
/// /etc/hostname at startup when unset.
pub host: Option<String>,
pub service: String,
}
impl Default for AgentConfig {
fn default() -> Self {
Self {
host: None,
service: "default".to_string(),
}
}
}
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "lowercase", tag = "kind")]
pub enum SourceConfig {
Journald {
#[serde(default)]
unit: Option<String>,
},
File {
path: PathBuf,
#[serde(default)]
from_beginning: bool,
},
}
impl Default for SourceConfig {
fn default() -> Self {
SourceConfig::Journald { unit: None }
}
}
#[derive(Debug, Clone, Deserialize)]
#[serde(default)]
pub struct BatchConfig {
pub max_size: usize,
pub flush_interval_ms: u64,
}
impl Default for BatchConfig {
fn default() -> Self {
Self {
max_size: 500,
flush_interval_ms: 2000,
}
}
}
#[derive(Debug, Clone, Deserialize)]
#[serde(default)]
pub struct IngestConfig {
pub endpoint: String,
}
impl Default for IngestConfig {
fn default() -> Self {
Self {
endpoint: "https://127.0.0.1:4317".to_string(),
}
}
}
#[derive(Debug, Clone, Deserialize)]
#[serde(default)]
pub struct TlsConfig {
pub ca_cert: PathBuf,
pub client_cert: PathBuf,
pub client_key: PathBuf,
}
impl Default for TlsConfig {
fn default() -> Self {
Self {
ca_cert: PathBuf::from("/etc/sentry-agent/ca.pem"),
client_cert: PathBuf::from("/etc/sentry-agent/client.pem"),
client_key: PathBuf::from("/etc/sentry-agent/client-key.pem"),
}
}
}
+45
View File
@@ -0,0 +1,45 @@
use crate::config::{IngestConfig, TlsConfig};
use crate::pb::{log_ingest_client::LogIngestClient, LogRecord, PushBatchRequest};
use anyhow::{Context, Result};
use tonic::transport::{Certificate, Channel, ClientTlsConfig, Identity};
/// Establishes an mTLS gRPC channel to the ingest service. Agents never
/// talk to Redpanda directly — this is the only network egress the agent
/// has, by design (see /docs/architecture.md).
pub async fn connect(ingest: &IngestConfig, tls: &TlsConfig) -> Result<LogIngestClient<Channel>> {
let ca = tokio::fs::read(&tls.ca_cert)
.await
.with_context(|| format!("reading CA cert at {}", tls.ca_cert.display()))?;
let cert = tokio::fs::read(&tls.client_cert)
.await
.with_context(|| format!("reading client cert at {}", tls.client_cert.display()))?;
let key = tokio::fs::read(&tls.client_key)
.await
.with_context(|| format!("reading client key at {}", tls.client_key.display()))?;
let tls_config = ClientTlsConfig::new()
.ca_certificate(Certificate::from_pem(ca))
.identity(Identity::from_pem(cert, key));
let channel = Channel::from_shared(ingest.endpoint.clone())
.context("invalid ingest endpoint URL")?
.tls_config(tls_config)
.context("configuring mTLS")?
.connect()
.await
.context("connecting to ingest service")?;
Ok(LogIngestClient::new(channel))
}
pub async fn send_batch(
client: &mut LogIngestClient<Channel>,
batch_id: String,
records: Vec<LogRecord>,
) -> Result<u32> {
let resp = client
.push_batch(PushBatchRequest { batch_id, records })
.await
.context("PushBatch RPC failed")?;
Ok(resp.into_inner().accepted)
}
+153
View File
@@ -0,0 +1,153 @@
mod batch;
mod config;
mod grpc;
mod source;
pub mod pb {
tonic::include_proto!("sentry.logs.v1");
}
use anyhow::{Context, Result};
use batch::Batcher;
use clap::Parser;
use config::Config;
use pb::{log_ingest_client::LogIngestClient, LogRecord, Severity};
use std::path::PathBuf;
use std::time::Duration;
use tokio::sync::mpsc;
use tonic::transport::Channel;
#[derive(Parser)]
#[command(name = "sentry-agent", about = "Sentry Linux log collector")]
struct Cli {
/// Path to a TOML config file. Defaults to /etc/sentry-agent/agent.toml
/// if present, otherwise built-in defaults (journald source, default
/// TLS cert paths under /etc/sentry-agent/).
#[arg(long)]
config: Option<PathBuf>,
}
#[tokio::main]
async fn main() -> Result<()> {
tracing_subscriber::fmt()
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
.init();
let cli = Cli::parse();
let cfg = Config::load(cli.config.as_deref()).context("loading config")?;
let host = cfg.agent.host.clone().unwrap_or_else(default_hostname);
let service = cfg.agent.service.clone();
let (tx, mut rx) = mpsc::channel(1024);
let source_handle = tokio::spawn(spawn_source(cfg.source.clone(), tx));
let mut client = grpc::connect(&cfg.ingest, &cfg.tls)
.await
.context("connecting to ingest service")?;
tracing::info!(endpoint = %cfg.ingest.endpoint, "connected to ingest service");
let flush_interval = Duration::from_millis(cfg.batch.flush_interval_ms);
let mut batcher = Batcher::new(cfg.batch.max_size, flush_interval);
let mut ticker = tokio::time::interval(flush_interval.max(Duration::from_millis(50)));
loop {
tokio::select! {
maybe_line = rx.recv() => {
let Some(raw) = maybe_line else {
tracing::warn!("source exited, flushing remaining batch and shutting down");
break;
};
let parsed = sentry_parser::parse(&raw.line);
let severity = to_pb_severity(raw.severity_hint.or(parsed.severity));
let record = LogRecord {
timestamp_unix_nano: raw.timestamp_unix_nano,
host: host.clone(),
service: service.clone(),
severity: severity as i32,
message: parsed.message,
attributes: parsed.attributes.into_iter().collect(),
};
if let Some(batch) = batcher.push(record) {
flush(&mut client, batch).await;
}
}
_ = ticker.tick() => {
if let Some(batch) = batcher.poll_timeout() {
flush(&mut client, batch).await;
}
}
}
}
if let Some(batch) = batcher.poll_timeout() {
flush(&mut client, batch).await;
}
source_handle.abort();
Ok(())
}
async fn spawn_source(source: config::SourceConfig, tx: source::LineSender) {
let result = match source {
#[cfg(feature = "journald")]
config::SourceConfig::Journald { unit } => source::journald::run(unit.as_deref(), tx).await,
#[cfg(not(feature = "journald"))]
config::SourceConfig::Journald { .. } => {
Err(anyhow::anyhow!("this build was compiled without the `journald` feature"))
}
#[cfg(feature = "file-tail")]
config::SourceConfig::File { path, from_beginning } => {
source::file_tail::run(&path, from_beginning, tx).await
}
#[cfg(not(feature = "file-tail"))]
config::SourceConfig::File { .. } => {
Err(anyhow::anyhow!("this build was compiled without the `file-tail` feature"))
}
};
if let Err(e) = result {
tracing::error!(error = %e, "log source exited with error");
}
}
async fn flush(client: &mut LogIngestClient<Channel>, batch: Vec<LogRecord>) {
let n = batch.len();
let batch_id = batch_id();
match grpc::send_batch(client, batch_id, batch).await {
Ok(accepted) => tracing::debug!(accepted, sent = n, "batch flushed"),
Err(e) => tracing::error!(error = %e, sent = n, "batch flush failed"),
}
}
/// Best-effort batch identifier for ingest-side dedup on retry. Not
/// globally unique (host + nanosecond timestamp), which is good enough for
/// Phase 0's single-agent-per-host reality; revisit if agents ever share
/// an identity or clock resolution becomes a problem.
fn batch_id() -> String {
let nanos = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0);
format!("{nanos:x}")
}
fn to_pb_severity(sev: Option<u8>) -> Severity {
match sev {
Some(0..=2) => Severity::Fatal, // emerg / alert / crit
Some(3) => Severity::Error, // err
Some(4) => Severity::Warn, // warning
Some(5) | Some(6) => Severity::Info, // notice / info
Some(7) => Severity::Debug, // debug
_ => Severity::Unspecified,
}
}
fn default_hostname() -> String {
if let Ok(s) = std::fs::read_to_string("/etc/hostname") {
let s = s.trim().to_string();
if !s.is_empty() {
return s;
}
}
std::env::var("HOSTNAME").unwrap_or_else(|_| "unknown-host".to_string())
}
@@ -0,0 +1,73 @@
use super::{LineSender, RawLine};
use anyhow::{Context, Result};
use std::io::SeekFrom;
use std::path::Path;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use tokio::fs::File;
use tokio::io::{AsyncBufReadExt, AsyncSeekExt, BufReader};
const POLL_INTERVAL: Duration = Duration::from_millis(500);
/// Polling-based file tailer: no inotify/`notify` crate dependency. Good
/// enough for Phase 0 (journald is the primary source). Handles basic
/// truncation (e.g. logrotate `copytruncate`) by detecting the file shrank
/// and reopening from the start. Does not follow rename-based rotation
/// (logrotate `create`) — that's deferred until file-tail is more than a
/// fallback path.
pub async fn run(path: &Path, from_beginning: bool, tx: LineSender) -> Result<()> {
let file = File::open(path)
.await
.with_context(|| format!("opening {}", path.display()))?;
let mut pos = if from_beginning { 0 } else { file.metadata().await?.len() };
let mut reader = BufReader::new(file);
reader.seek(SeekFrom::Start(pos)).await?;
let mut buf = String::new();
loop {
buf.clear();
let n = reader
.read_line(&mut buf)
.await
.context("reading line from file")?;
if n == 0 {
let metadata = tokio::fs::metadata(path).await.context("stat-ing file")?;
if metadata.len() < pos {
tracing::warn!(path = %path.display(), "file shrank, assuming truncation and reopening from start");
let f = File::open(path)
.await
.context("reopening file after truncation")?;
reader = BufReader::new(f);
pos = 0;
}
tokio::time::sleep(POLL_INTERVAL).await;
continue;
}
pos += n as u64;
let line = buf.trim_end_matches(['\n', '\r']).to_string();
if line.is_empty() {
continue;
}
let timestamp_unix_nano = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_nanos() as i64)
.unwrap_or(0);
if tx
.send(RawLine {
line,
timestamp_unix_nano,
severity_hint: None,
})
.await
.is_err()
{
break;
}
}
Ok(())
}
+75
View File
@@ -0,0 +1,75 @@
use super::{LineSender, RawLine};
use anyhow::{Context, Result};
use std::time::{SystemTime, UNIX_EPOCH};
use tokio::io::{AsyncBufReadExt, BufReader};
use tokio::process::Command;
/// Reads journald entries by shelling out to `journalctl -f -o json`
/// rather than linking libsystemd via FFI. Linking libsystemd into a
/// statically-linked musl binary is fragile (it pulls in dbus/libcap
/// transitively and isn't designed for static linking) and would work
/// against the no-glibc-runtime-deps constraint in spirit even where it's
/// technically possible. `journalctl` ships on every systemd distro this
/// agent targets, so shelling out avoids the problem entirely. See
/// /docs/architecture.md.
pub async fn run(unit: Option<&str>, tx: LineSender) -> Result<()> {
let mut cmd = Command::new("journalctl");
cmd.arg("-f")
.arg("-o")
.arg("json")
.arg("--since=now")
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::null());
if let Some(unit) = unit {
cmd.arg("-u").arg(unit);
}
let mut child = cmd
.spawn()
.context("spawning journalctl -f -o json (is systemd-journal installed?)")?;
let stdout = child.stdout.take().context("journalctl child had no stdout")?;
let mut lines = BufReader::new(stdout).lines();
while let Some(line) = lines.next_line().await.context("reading journalctl output")? {
let Ok(entry) = serde_json::from_str::<serde_json::Value>(&line) else {
tracing::warn!(%line, "skipping unparseable journalctl JSON line");
continue;
};
let message = entry
.get("MESSAGE")
.and_then(|v| v.as_str())
.unwrap_or_default()
.to_string();
if message.is_empty() {
continue;
}
let severity_hint = entry
.get("PRIORITY")
.and_then(|v| v.as_str().map(str::to_string).or_else(|| v.as_u64().map(|n| n.to_string())))
.and_then(|s| s.parse::<u8>().ok())
.filter(|&p| p <= 7);
let timestamp_unix_nano = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_nanos() as i64)
.unwrap_or(0);
if tx
.send(RawLine {
line: message,
timestamp_unix_nano,
severity_hint,
})
.await
.is_err()
{
break; // receiver dropped, agent is shutting down
}
}
let status = child.wait().await.context("waiting for journalctl to exit")?;
tracing::warn!(?status, "journalctl exited");
Ok(())
}
+23
View File
@@ -0,0 +1,23 @@
use tokio::sync::mpsc;
/// A raw line read from a source, plus whatever metadata the source itself
/// already knows before the RFC 5424 parser ever sees it.
#[derive(Debug, Clone)]
pub struct RawLine {
pub line: String,
/// Unix epoch nanoseconds at time of read.
pub timestamp_unix_nano: i64,
/// Syslog severity (0-7) if the source already knows it independent of
/// the line's own content — e.g. journald's PRIORITY field. When set,
/// this takes precedence over whatever the RFC 5424 parser infers from
/// the message text, since it comes from a more authoritative place.
pub severity_hint: Option<u8>,
}
pub type LineSender = mpsc::Sender<RawLine>;
#[cfg(feature = "journald")]
pub mod journald;
#[cfg(feature = "file-tail")]
pub mod file_tail;