Scaffold Phase 0: agent -> Redpanda -> ingest -> ClickHouse -> api -> web

End-to-end log pipeline for Linux hosts, per /docs/architecture.md: - proto: shared gRPC contract (agent <-> ingest), Go bindings checked in - agent: Rust, musl-targeted, journald/file sourcing, RFC5424 parser, mTLS gRPC client, no required config for the common case - ingest: Go, single binary with --mode server|consumer|all; gRPC front end forwards to Redpanda unchanged, consumer normalizes and batch-writes to ClickHouse with at-least-once delivery - storage: ClickHouse schema + a plain SQL-file migration runner - api: minimal SELECT-only query endpoint, plain REST (not gRPC+gateway yet -- see api/README.md) - web: SvelteKit static SPA, one query page - transport: Redpanda compose + topic provisioning - cli: sentryctl ping stub - hack/dev-certs: throwaway CA + cert generation for local mTLS - root docker-compose.yml + docs/phase-0-runbook.md tie it together Not yet run end-to-end against real Docker/ClickHouse/Redpanda -- see the runbook's caveats section before relying on this working as-is.
2026-08-13 08:25:19 -07:00
commit b6b092c912
92 changed files with 7796 additions and 0 deletions
@@ -0,0 +1,73 @@
+use super::{LineSender, RawLine};
+use anyhow::{Context, Result};
+use std::io::SeekFrom;
+use std::path::Path;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use tokio::fs::File;
+use tokio::io::{AsyncBufReadExt, AsyncSeekExt, BufReader};
+
+const POLL_INTERVAL: Duration = Duration::from_millis(500);
+
+/// Polling-based file tailer: no inotify/`notify` crate dependency. Good
+/// enough for Phase 0 (journald is the primary source). Handles basic
+/// truncation (e.g. logrotate `copytruncate`) by detecting the file shrank
+/// and reopening from the start. Does not follow rename-based rotation
+/// (logrotate `create`) — that's deferred until file-tail is more than a
+/// fallback path.
+pub async fn run(path: &Path, from_beginning: bool, tx: LineSender) -> Result<()> {
+    let file = File::open(path)
+        .await
+        .with_context(|| format!("opening {}", path.display()))?;
+
+    let mut pos = if from_beginning { 0 } else { file.metadata().await?.len() };
+
+    let mut reader = BufReader::new(file);
+    reader.seek(SeekFrom::Start(pos)).await?;
+    let mut buf = String::new();
+
+    loop {
+        buf.clear();
+        let n = reader
+            .read_line(&mut buf)
+            .await
+            .context("reading line from file")?;
+        if n == 0 {
+            let metadata = tokio::fs::metadata(path).await.context("stat-ing file")?;
+            if metadata.len() < pos {
+                tracing::warn!(path = %path.display(), "file shrank, assuming truncation and reopening from start");
+                let f = File::open(path)
+                    .await
+                    .context("reopening file after truncation")?;
+                reader = BufReader::new(f);
+                pos = 0;
+            }
+            tokio::time::sleep(POLL_INTERVAL).await;
+            continue;
+        }
+        pos += n as u64;
+
+        let line = buf.trim_end_matches(['\n', '\r']).to_string();
+        if line.is_empty() {
+            continue;
+        }
+
+        let timestamp_unix_nano = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .map(|d| d.as_nanos() as i64)
+            .unwrap_or(0);
+
+        if tx
+            .send(RawLine {
+                line,
+                timestamp_unix_nano,
+                severity_hint: None,
+            })
+            .await
+            .is_err()
+        {
+            break;
+        }
+    }
+
+    Ok(())
+}
@@ -0,0 +1,75 @@
+use super::{LineSender, RawLine};
+use anyhow::{Context, Result};
+use std::time::{SystemTime, UNIX_EPOCH};
+use tokio::io::{AsyncBufReadExt, BufReader};
+use tokio::process::Command;
+
+/// Reads journald entries by shelling out to `journalctl -f -o json`
+/// rather than linking libsystemd via FFI. Linking libsystemd into a
+/// statically-linked musl binary is fragile (it pulls in dbus/libcap
+/// transitively and isn't designed for static linking) and would work
+/// against the no-glibc-runtime-deps constraint in spirit even where it's
+/// technically possible. `journalctl` ships on every systemd distro this
+/// agent targets, so shelling out avoids the problem entirely. See
+/// /docs/architecture.md.
+pub async fn run(unit: Option<&str>, tx: LineSender) -> Result<()> {
+    let mut cmd = Command::new("journalctl");
+    cmd.arg("-f")
+        .arg("-o")
+        .arg("json")
+        .arg("--since=now")
+        .stdout(std::process::Stdio::piped())
+        .stderr(std::process::Stdio::null());
+    if let Some(unit) = unit {
+        cmd.arg("-u").arg(unit);
+    }
+
+    let mut child = cmd
+        .spawn()
+        .context("spawning journalctl -f -o json (is systemd-journal installed?)")?;
+    let stdout = child.stdout.take().context("journalctl child had no stdout")?;
+    let mut lines = BufReader::new(stdout).lines();
+
+    while let Some(line) = lines.next_line().await.context("reading journalctl output")? {
+        let Ok(entry) = serde_json::from_str::<serde_json::Value>(&line) else {
+            tracing::warn!(%line, "skipping unparseable journalctl JSON line");
+            continue;
+        };
+
+        let message = entry
+            .get("MESSAGE")
+            .and_then(|v| v.as_str())
+            .unwrap_or_default()
+            .to_string();
+        if message.is_empty() {
+            continue;
+        }
+
+        let severity_hint = entry
+            .get("PRIORITY")
+            .and_then(|v| v.as_str().map(str::to_string).or_else(|| v.as_u64().map(|n| n.to_string())))
+            .and_then(|s| s.parse::<u8>().ok())
+            .filter(|&p| p <= 7);
+
+        let timestamp_unix_nano = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .map(|d| d.as_nanos() as i64)
+            .unwrap_or(0);
+
+        if tx
+            .send(RawLine {
+                line: message,
+                timestamp_unix_nano,
+                severity_hint,
+            })
+            .await
+            .is_err()
+        {
+            break; // receiver dropped, agent is shutting down
+        }
+    }
+
+    let status = child.wait().await.context("waiting for journalctl to exit")?;
+    tracing::warn!(?status, "journalctl exited");
+    Ok(())
+}
@@ -0,0 +1,23 @@
+use tokio::sync::mpsc;
+
+/// A raw line read from a source, plus whatever metadata the source itself
+/// already knows before the RFC 5424 parser ever sees it.
+#[derive(Debug, Clone)]
+pub struct RawLine {
+    pub line: String,
+    /// Unix epoch nanoseconds at time of read.
+    pub timestamp_unix_nano: i64,
+    /// Syslog severity (0-7) if the source already knows it independent of
+    /// the line's own content — e.g. journald's PRIORITY field. When set,
+    /// this takes precedence over whatever the RFC 5424 parser infers from
+    /// the message text, since it comes from a more authoritative place.
+    pub severity_hint: Option<u8>,
+}
+
+pub type LineSender = mpsc::Sender<RawLine>;
+
+#[cfg(feature = "journald")]
+pub mod journald;
+
+#[cfg(feature = "file-tail")]
+pub mod file_tail;