Scaffold Phase 0: agent -> Redpanda -> ingest -> ClickHouse -> api -> web

End-to-end log pipeline for Linux hosts, per /docs/architecture.md:

- proto: shared gRPC contract (agent <-> ingest), Go bindings checked in
- agent: Rust, musl-targeted, journald/file sourcing, RFC5424 parser,
  mTLS gRPC client, no required config for the common case
- ingest: Go, single binary with --mode server|consumer|all; gRPC front
  end forwards to Redpanda unchanged, consumer normalizes and
  batch-writes to ClickHouse with at-least-once delivery
- storage: ClickHouse schema + a plain SQL-file migration runner
- api: minimal SELECT-only query endpoint, plain REST (not gRPC+gateway
  yet -- see api/README.md)
- web: SvelteKit static SPA, one query page
- transport: Redpanda compose + topic provisioning
- cli: sentryctl ping stub
- hack/dev-certs: throwaway CA + cert generation for local mTLS
- root docker-compose.yml + docs/phase-0-runbook.md tie it together

Not yet run end-to-end against real Docker/ClickHouse/Redpanda -- see the
runbook's caveats section before relying on this working as-is.
This commit is contained in:
2026-08-13 08:25:19 -07:00
commit b6b092c912
92 changed files with 7796 additions and 0 deletions
+95
View File
@@ -0,0 +1,95 @@
// Package config loads ingest's configuration from environment variables.
// Phase 0 deliberately has no config file format of its own — env vars are
// enough for a docker-compose/k8s deployment and avoid pulling in a config
// library.
package config
import (
"fmt"
"os"
"strconv"
"strings"
)
type Config struct {
GRPC GRPCConfig
TLS TLSConfig
Redpanda RedpandaConfig
ClickHouse ClickHouseConfig
Batch BatchConfig
}
type GRPCConfig struct {
ListenAddr string
}
// TLSConfig is the server-side mTLS material: the ingest service's own
// cert/key, and the CA used to verify agent client certs.
type TLSConfig struct {
CertFile string
KeyFile string
ClientCAFile string
}
type RedpandaConfig struct {
Brokers []string
Topic string
ConsumerGroup string
}
type ClickHouseConfig struct {
Addr string
Database string
Username string
Password string
}
type BatchConfig struct {
MaxSize int
FlushIntervalMS int
}
func Load() (Config, error) {
cfg := Config{
GRPC: GRPCConfig{
ListenAddr: getenv("GRPC_LISTEN_ADDR", ":4317"),
},
TLS: TLSConfig{
CertFile: getenv("TLS_CERT_FILE", "/etc/sentry-ingest/server.pem"),
KeyFile: getenv("TLS_KEY_FILE", "/etc/sentry-ingest/server-key.pem"),
ClientCAFile: getenv("TLS_CLIENT_CA_FILE", "/etc/sentry-ingest/ca.pem"),
},
Redpanda: RedpandaConfig{
Brokers: strings.Split(getenv("REDPANDA_BROKERS", "localhost:9092"), ","),
Topic: getenv("REDPANDA_TOPIC", "sentry.logs.raw"),
ConsumerGroup: getenv("REDPANDA_CONSUMER_GROUP", "sentry-ingest"),
},
ClickHouse: ClickHouseConfig{
Addr: getenv("CLICKHOUSE_ADDR", "localhost:9000"),
Database: getenv("CLICKHOUSE_DATABASE", "sentry"),
Username: getenv("CLICKHOUSE_USERNAME", "default"),
Password: getenv("CLICKHOUSE_PASSWORD", ""),
},
}
maxSize, err := strconv.Atoi(getenv("CONSUMER_BATCH_MAX_SIZE", "500"))
if err != nil {
return Config{}, fmt.Errorf("CONSUMER_BATCH_MAX_SIZE: %w", err)
}
cfg.Batch.MaxSize = maxSize
flushMS, err := strconv.Atoi(getenv("CONSUMER_BATCH_FLUSH_INTERVAL_MS", "2000"))
if err != nil {
return Config{}, fmt.Errorf("CONSUMER_BATCH_FLUSH_INTERVAL_MS: %w", err)
}
cfg.Batch.FlushIntervalMS = flushMS
return cfg, nil
}
func getenv(key, fallback string) string {
if v := os.Getenv(key); v != "" {
return v
}
return fallback
}