Scaffold Phase 0: agent -> Redpanda -> ingest -> ClickHouse -> api -> web
End-to-end log pipeline for Linux hosts, per /docs/architecture.md: - proto: shared gRPC contract (agent <-> ingest), Go bindings checked in - agent: Rust, musl-targeted, journald/file sourcing, RFC5424 parser, mTLS gRPC client, no required config for the common case - ingest: Go, single binary with --mode server|consumer|all; gRPC front end forwards to Redpanda unchanged, consumer normalizes and batch-writes to ClickHouse with at-least-once delivery - storage: ClickHouse schema + a plain SQL-file migration runner - api: minimal SELECT-only query endpoint, plain REST (not gRPC+gateway yet -- see api/README.md) - web: SvelteKit static SPA, one query page - transport: Redpanda compose + topic provisioning - cli: sentryctl ping stub - hack/dev-certs: throwaway CA + cert generation for local mTLS - root docker-compose.yml + docs/phase-0-runbook.md tie it together Not yet run end-to-end against real Docker/ClickHouse/Redpanda -- see the runbook's caveats section before relying on this working as-is.
This commit is contained in:
@@ -0,0 +1,95 @@
|
||||
// Package config loads ingest's configuration from environment variables.
|
||||
// Phase 0 deliberately has no config file format of its own — env vars are
|
||||
// enough for a docker-compose/k8s deployment and avoid pulling in a config
|
||||
// library.
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
GRPC GRPCConfig
|
||||
TLS TLSConfig
|
||||
Redpanda RedpandaConfig
|
||||
ClickHouse ClickHouseConfig
|
||||
Batch BatchConfig
|
||||
}
|
||||
|
||||
type GRPCConfig struct {
|
||||
ListenAddr string
|
||||
}
|
||||
|
||||
// TLSConfig is the server-side mTLS material: the ingest service's own
|
||||
// cert/key, and the CA used to verify agent client certs.
|
||||
type TLSConfig struct {
|
||||
CertFile string
|
||||
KeyFile string
|
||||
ClientCAFile string
|
||||
}
|
||||
|
||||
type RedpandaConfig struct {
|
||||
Brokers []string
|
||||
Topic string
|
||||
ConsumerGroup string
|
||||
}
|
||||
|
||||
type ClickHouseConfig struct {
|
||||
Addr string
|
||||
Database string
|
||||
Username string
|
||||
Password string
|
||||
}
|
||||
|
||||
type BatchConfig struct {
|
||||
MaxSize int
|
||||
FlushIntervalMS int
|
||||
}
|
||||
|
||||
func Load() (Config, error) {
|
||||
cfg := Config{
|
||||
GRPC: GRPCConfig{
|
||||
ListenAddr: getenv("GRPC_LISTEN_ADDR", ":4317"),
|
||||
},
|
||||
TLS: TLSConfig{
|
||||
CertFile: getenv("TLS_CERT_FILE", "/etc/sentry-ingest/server.pem"),
|
||||
KeyFile: getenv("TLS_KEY_FILE", "/etc/sentry-ingest/server-key.pem"),
|
||||
ClientCAFile: getenv("TLS_CLIENT_CA_FILE", "/etc/sentry-ingest/ca.pem"),
|
||||
},
|
||||
Redpanda: RedpandaConfig{
|
||||
Brokers: strings.Split(getenv("REDPANDA_BROKERS", "localhost:9092"), ","),
|
||||
Topic: getenv("REDPANDA_TOPIC", "sentry.logs.raw"),
|
||||
ConsumerGroup: getenv("REDPANDA_CONSUMER_GROUP", "sentry-ingest"),
|
||||
},
|
||||
ClickHouse: ClickHouseConfig{
|
||||
Addr: getenv("CLICKHOUSE_ADDR", "localhost:9000"),
|
||||
Database: getenv("CLICKHOUSE_DATABASE", "sentry"),
|
||||
Username: getenv("CLICKHOUSE_USERNAME", "default"),
|
||||
Password: getenv("CLICKHOUSE_PASSWORD", ""),
|
||||
},
|
||||
}
|
||||
|
||||
maxSize, err := strconv.Atoi(getenv("CONSUMER_BATCH_MAX_SIZE", "500"))
|
||||
if err != nil {
|
||||
return Config{}, fmt.Errorf("CONSUMER_BATCH_MAX_SIZE: %w", err)
|
||||
}
|
||||
cfg.Batch.MaxSize = maxSize
|
||||
|
||||
flushMS, err := strconv.Atoi(getenv("CONSUMER_BATCH_FLUSH_INTERVAL_MS", "2000"))
|
||||
if err != nil {
|
||||
return Config{}, fmt.Errorf("CONSUMER_BATCH_FLUSH_INTERVAL_MS: %w", err)
|
||||
}
|
||||
cfg.Batch.FlushIntervalMS = flushMS
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func getenv(key, fallback string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
Reference in New Issue
Block a user