Scaffold Phase 0: agent -> Redpanda -> ingest -> ClickHouse -> api -> web
End-to-end log pipeline for Linux hosts, per /docs/architecture.md: - proto: shared gRPC contract (agent <-> ingest), Go bindings checked in - agent: Rust, musl-targeted, journald/file sourcing, RFC5424 parser, mTLS gRPC client, no required config for the common case - ingest: Go, single binary with --mode server|consumer|all; gRPC front end forwards to Redpanda unchanged, consumer normalizes and batch-writes to ClickHouse with at-least-once delivery - storage: ClickHouse schema + a plain SQL-file migration runner - api: minimal SELECT-only query endpoint, plain REST (not gRPC+gateway yet -- see api/README.md) - web: SvelteKit static SPA, one query page - transport: Redpanda compose + topic provisioning - cli: sentryctl ping stub - hack/dev-certs: throwaway CA + cert generation for local mTLS - root docker-compose.yml + docs/phase-0-runbook.md tie it together Not yet run end-to-end against real Docker/ClickHouse/Redpanda -- see the runbook's caveats section before relying on this working as-is.
This commit is contained in:
@@ -0,0 +1,45 @@
|
||||
use crate::config::{IngestConfig, TlsConfig};
|
||||
use crate::pb::{log_ingest_client::LogIngestClient, LogRecord, PushBatchRequest};
|
||||
use anyhow::{Context, Result};
|
||||
use tonic::transport::{Certificate, Channel, ClientTlsConfig, Identity};
|
||||
|
||||
/// Establishes an mTLS gRPC channel to the ingest service. Agents never
|
||||
/// talk to Redpanda directly — this is the only network egress the agent
|
||||
/// has, by design (see /docs/architecture.md).
|
||||
pub async fn connect(ingest: &IngestConfig, tls: &TlsConfig) -> Result<LogIngestClient<Channel>> {
|
||||
let ca = tokio::fs::read(&tls.ca_cert)
|
||||
.await
|
||||
.with_context(|| format!("reading CA cert at {}", tls.ca_cert.display()))?;
|
||||
let cert = tokio::fs::read(&tls.client_cert)
|
||||
.await
|
||||
.with_context(|| format!("reading client cert at {}", tls.client_cert.display()))?;
|
||||
let key = tokio::fs::read(&tls.client_key)
|
||||
.await
|
||||
.with_context(|| format!("reading client key at {}", tls.client_key.display()))?;
|
||||
|
||||
let tls_config = ClientTlsConfig::new()
|
||||
.ca_certificate(Certificate::from_pem(ca))
|
||||
.identity(Identity::from_pem(cert, key));
|
||||
|
||||
let channel = Channel::from_shared(ingest.endpoint.clone())
|
||||
.context("invalid ingest endpoint URL")?
|
||||
.tls_config(tls_config)
|
||||
.context("configuring mTLS")?
|
||||
.connect()
|
||||
.await
|
||||
.context("connecting to ingest service")?;
|
||||
|
||||
Ok(LogIngestClient::new(channel))
|
||||
}
|
||||
|
||||
pub async fn send_batch(
|
||||
client: &mut LogIngestClient<Channel>,
|
||||
batch_id: String,
|
||||
records: Vec<LogRecord>,
|
||||
) -> Result<u32> {
|
||||
let resp = client
|
||||
.push_batch(PushBatchRequest { batch_id, records })
|
||||
.await
|
||||
.context("PushBatch RPC failed")?;
|
||||
Ok(resp.into_inner().accepted)
|
||||
}
|
||||
Reference in New Issue
Block a user