jcoffey-dev
cd8aa290ca
Extends the agent, ingest, storage, api, and web with Windows Event Log/ETW sourcing and Tantivy-backed free-text search, per the approved Phase 1 plan. - CLAUDE.md: materialized on disk (never existed as a file before) with a new Phase 1 "done looks like" section. - agent: Windows Event Log (EvtSubscribe) and ETW sources, Windows service wrapper (install/uninstall/run-service), both feature- and target_os-gated so Linux builds/tests/clippy stay unaffected. Also fixed two pre-existing Phase 0 clippy gaps (dead-code on default-features-only builds, a type-inference edge case) found while testing every feature combination properly for the first time. UNVERIFIED on real Windows -- no Windows toolchain existed anywhere in the build environment; flagged prominently in three places. - proto/ingest: new record_id field, assigned once server-side in ingest's gRPC front end so ClickHouse and Tantivy agree on the same ID for the same record. - storage: record_id column + bloom filter index, verified against a live ClickHouse. - search: new service, Tantivy index, rskafka consumer as an independent second consumer group on the same Redpanda topic ingest already reads. - api/web: new /search endpoint and page, sharing the query page's result-table shape and component. - hack/windows-fixture: sends realistic Windows-shaped data straight to ingest, so the pipeline's handling of it is verifiable without a Windows host. Verified end-to-end on the live docker-compose stack: the same record_id comes back from both /query and /search for the same log line, including for windows-fixture's synthetic Windows Event Log data. Real bugs found and fixed along the way: api/Dockerfile missing proto/ in its build context, search's logs being completely silent (RUST_LOG gap), and search/target/ missing from .gitignore/.dockerignore.
94 lines
3.4 KiB
Rust
94 lines
3.4 KiB
Rust
use anyhow::{Context, Result};
|
|
use std::collections::HashMap;
|
|
use std::path::{Path, PathBuf};
|
|
|
|
/// Tracks per-partition offsets in a plain JSON file next to the Tantivy
|
|
/// index, since rskafka is a low-level client with no built-in consumer-
|
|
/// group coordination/offset-commit protocol (unlike kafka-go on the
|
|
/// ingest side) -- there's no broker-side group to commit to here, so
|
|
/// this service owns its own offset bookkeeping.
|
|
///
|
|
/// Best-effort, not exactly-once: if the process dies between processing
|
|
/// a record and persisting its offset, that record gets reprocessed on
|
|
/// restart. This is fine because `SearchIndex::upsert` is delete-then-add
|
|
/// on `record_id` -- reprocessing the same record overwrites the same
|
|
/// document rather than duplicating it.
|
|
#[derive(Debug)]
|
|
pub struct OffsetStore {
|
|
path: PathBuf,
|
|
offsets: HashMap<i32, i64>,
|
|
}
|
|
|
|
impl OffsetStore {
|
|
pub async fn load(path: &Path) -> Result<Self> {
|
|
let offsets = match tokio::fs::read(path).await {
|
|
Ok(bytes) => serde_json::from_slice(&bytes)
|
|
.with_context(|| format!("parsing offsets file {}", path.display()))?,
|
|
Err(e) if e.kind() == std::io::ErrorKind::NotFound => HashMap::new(),
|
|
Err(e) => return Err(e).with_context(|| format!("reading offsets file {}", path.display())),
|
|
};
|
|
Ok(Self {
|
|
path: path.to_path_buf(),
|
|
offsets,
|
|
})
|
|
}
|
|
|
|
/// Next offset to fetch for a partition -- 0 (earliest) if never
|
|
/// recorded before.
|
|
pub fn get(&self, partition: i32) -> i64 {
|
|
self.offsets.get(&partition).copied().unwrap_or(0)
|
|
}
|
|
|
|
pub fn set(&mut self, partition: i32, offset: i64) {
|
|
self.offsets.insert(partition, offset);
|
|
}
|
|
|
|
pub async fn persist(&self) -> Result<()> {
|
|
if let Some(parent) = self.path.parent() {
|
|
tokio::fs::create_dir_all(parent)
|
|
.await
|
|
.with_context(|| format!("creating offsets directory {}", parent.display()))?;
|
|
}
|
|
let bytes = serde_json::to_vec_pretty(&self.offsets).context("serializing offsets")?;
|
|
tokio::fs::write(&self.path, bytes)
|
|
.await
|
|
.with_context(|| format!("writing offsets file {}", self.path.display()))
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[tokio::test]
|
|
async fn get_defaults_to_zero_for_unknown_partition() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let store = OffsetStore::load(&dir.path().join("offsets.json")).await.unwrap();
|
|
assert_eq!(store.get(0), 0);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn missing_file_loads_as_empty_not_an_error() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("does-not-exist.json");
|
|
let store = OffsetStore::load(&path).await;
|
|
assert!(store.is_ok(), "expected a missing offsets file to load as empty, got {store:?}");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn persist_and_reload_round_trips() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("offsets.json");
|
|
|
|
let mut store = OffsetStore::load(&path).await.unwrap();
|
|
store.set(0, 42);
|
|
store.set(1, 7);
|
|
store.persist().await.unwrap();
|
|
|
|
let reloaded = OffsetStore::load(&path).await.unwrap();
|
|
assert_eq!(reloaded.get(0), 42);
|
|
assert_eq!(reloaded.get(1), 7);
|
|
assert_eq!(reloaded.get(2), 0, "unset partition should still default to 0");
|
|
}
|
|
}
|