Scaffold Phase 0: agent -> Redpanda -> ingest -> ClickHouse -> api -> web
End-to-end log pipeline for Linux hosts, per /docs/architecture.md: - proto: shared gRPC contract (agent <-> ingest), Go bindings checked in - agent: Rust, musl-targeted, journald/file sourcing, RFC5424 parser, mTLS gRPC client, no required config for the common case - ingest: Go, single binary with --mode server|consumer|all; gRPC front end forwards to Redpanda unchanged, consumer normalizes and batch-writes to ClickHouse with at-least-once delivery - storage: ClickHouse schema + a plain SQL-file migration runner - api: minimal SELECT-only query endpoint, plain REST (not gRPC+gateway yet -- see api/README.md) - web: SvelteKit static SPA, one query page - transport: Redpanda compose + topic provisioning - cli: sentryctl ping stub - hack/dev-certs: throwaway CA + cert generation for local mTLS - root docker-compose.yml + docs/phase-0-runbook.md tie it together Not yet run end-to-end against real Docker/ClickHouse/Redpanda -- see the runbook's caveats section before relying on this working as-is.
This commit is contained in:
@@ -0,0 +1,60 @@
|
||||
// Package clickhousewriter batch-inserts normalized log rows into
|
||||
// ClickHouse using the native protocol driver's batch API.
|
||||
package clickhousewriter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2"
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
|
||||
"github.com/sentry/sentry/ingest/internal/config"
|
||||
"github.com/sentry/sentry/ingest/internal/normalize"
|
||||
logsv1 "github.com/sentry/sentry/proto/sentry/logs/v1"
|
||||
)
|
||||
|
||||
type Writer struct {
|
||||
conn driver.Conn
|
||||
}
|
||||
|
||||
func New(ctx context.Context, cfg config.ClickHouseConfig) (*Writer, error) {
|
||||
conn, err := clickhouse.Open(&clickhouse.Options{
|
||||
Addr: []string{cfg.Addr},
|
||||
Auth: clickhouse.Auth{
|
||||
Database: cfg.Database,
|
||||
Username: cfg.Username,
|
||||
Password: cfg.Password,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("opening clickhouse connection: %w", err)
|
||||
}
|
||||
if err := conn.Ping(ctx); err != nil {
|
||||
return nil, fmt.Errorf("pinging clickhouse: %w", err)
|
||||
}
|
||||
return &Writer{conn: conn}, nil
|
||||
}
|
||||
|
||||
func (w *Writer) Close() error {
|
||||
return w.conn.Close()
|
||||
}
|
||||
|
||||
func (w *Writer) WriteBatch(ctx context.Context, records []*logsv1.LogRecord) error {
|
||||
batch, err := w.conn.PrepareBatch(ctx, "INSERT INTO logs (timestamp, host, service, severity, message, attributes)")
|
||||
if err != nil {
|
||||
return fmt.Errorf("preparing batch: %w", err)
|
||||
}
|
||||
|
||||
for _, rec := range records {
|
||||
row := normalize.ToRow(rec)
|
||||
if err := batch.Append(row.Timestamp, row.Host, row.Service, row.Severity, row.Message, row.Attributes); err != nil {
|
||||
return fmt.Errorf("appending row to batch: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := batch.Send(); err != nil {
|
||||
return fmt.Errorf("sending batch: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
// Package config loads ingest's configuration from environment variables.
|
||||
// Phase 0 deliberately has no config file format of its own — env vars are
|
||||
// enough for a docker-compose/k8s deployment and avoid pulling in a config
|
||||
// library.
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
GRPC GRPCConfig
|
||||
TLS TLSConfig
|
||||
Redpanda RedpandaConfig
|
||||
ClickHouse ClickHouseConfig
|
||||
Batch BatchConfig
|
||||
}
|
||||
|
||||
type GRPCConfig struct {
|
||||
ListenAddr string
|
||||
}
|
||||
|
||||
// TLSConfig is the server-side mTLS material: the ingest service's own
|
||||
// cert/key, and the CA used to verify agent client certs.
|
||||
type TLSConfig struct {
|
||||
CertFile string
|
||||
KeyFile string
|
||||
ClientCAFile string
|
||||
}
|
||||
|
||||
type RedpandaConfig struct {
|
||||
Brokers []string
|
||||
Topic string
|
||||
ConsumerGroup string
|
||||
}
|
||||
|
||||
type ClickHouseConfig struct {
|
||||
Addr string
|
||||
Database string
|
||||
Username string
|
||||
Password string
|
||||
}
|
||||
|
||||
type BatchConfig struct {
|
||||
MaxSize int
|
||||
FlushIntervalMS int
|
||||
}
|
||||
|
||||
func Load() (Config, error) {
|
||||
cfg := Config{
|
||||
GRPC: GRPCConfig{
|
||||
ListenAddr: getenv("GRPC_LISTEN_ADDR", ":4317"),
|
||||
},
|
||||
TLS: TLSConfig{
|
||||
CertFile: getenv("TLS_CERT_FILE", "/etc/sentry-ingest/server.pem"),
|
||||
KeyFile: getenv("TLS_KEY_FILE", "/etc/sentry-ingest/server-key.pem"),
|
||||
ClientCAFile: getenv("TLS_CLIENT_CA_FILE", "/etc/sentry-ingest/ca.pem"),
|
||||
},
|
||||
Redpanda: RedpandaConfig{
|
||||
Brokers: strings.Split(getenv("REDPANDA_BROKERS", "localhost:9092"), ","),
|
||||
Topic: getenv("REDPANDA_TOPIC", "sentry.logs.raw"),
|
||||
ConsumerGroup: getenv("REDPANDA_CONSUMER_GROUP", "sentry-ingest"),
|
||||
},
|
||||
ClickHouse: ClickHouseConfig{
|
||||
Addr: getenv("CLICKHOUSE_ADDR", "localhost:9000"),
|
||||
Database: getenv("CLICKHOUSE_DATABASE", "sentry"),
|
||||
Username: getenv("CLICKHOUSE_USERNAME", "default"),
|
||||
Password: getenv("CLICKHOUSE_PASSWORD", ""),
|
||||
},
|
||||
}
|
||||
|
||||
maxSize, err := strconv.Atoi(getenv("CONSUMER_BATCH_MAX_SIZE", "500"))
|
||||
if err != nil {
|
||||
return Config{}, fmt.Errorf("CONSUMER_BATCH_MAX_SIZE: %w", err)
|
||||
}
|
||||
cfg.Batch.MaxSize = maxSize
|
||||
|
||||
flushMS, err := strconv.Atoi(getenv("CONSUMER_BATCH_FLUSH_INTERVAL_MS", "2000"))
|
||||
if err != nil {
|
||||
return Config{}, fmt.Errorf("CONSUMER_BATCH_FLUSH_INTERVAL_MS: %w", err)
|
||||
}
|
||||
cfg.Batch.FlushIntervalMS = flushMS
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func getenv(key, fallback string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package config
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestLoadDefaults(t *testing.T) {
|
||||
cfg, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
if cfg.GRPC.ListenAddr != ":4317" {
|
||||
t.Errorf("GRPC.ListenAddr = %q, want :4317", cfg.GRPC.ListenAddr)
|
||||
}
|
||||
if cfg.Redpanda.Topic != "sentry.logs.raw" {
|
||||
t.Errorf("Redpanda.Topic = %q, want sentry.logs.raw", cfg.Redpanda.Topic)
|
||||
}
|
||||
if cfg.Batch.MaxSize != 500 {
|
||||
t.Errorf("Batch.MaxSize = %d, want 500", cfg.Batch.MaxSize)
|
||||
}
|
||||
if cfg.Batch.FlushIntervalMS != 2000 {
|
||||
t.Errorf("Batch.FlushIntervalMS = %d, want 2000", cfg.Batch.FlushIntervalMS)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadOverridesFromEnv(t *testing.T) {
|
||||
t.Setenv("GRPC_LISTEN_ADDR", ":9999")
|
||||
t.Setenv("REDPANDA_BROKERS", "a:9092,b:9092")
|
||||
t.Setenv("CONSUMER_BATCH_MAX_SIZE", "10")
|
||||
|
||||
cfg, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
if cfg.GRPC.ListenAddr != ":9999" {
|
||||
t.Errorf("GRPC.ListenAddr = %q, want :9999", cfg.GRPC.ListenAddr)
|
||||
}
|
||||
if len(cfg.Redpanda.Brokers) != 2 || cfg.Redpanda.Brokers[0] != "a:9092" || cfg.Redpanda.Brokers[1] != "b:9092" {
|
||||
t.Errorf("Redpanda.Brokers = %+v, want [a:9092 b:9092]", cfg.Redpanda.Brokers)
|
||||
}
|
||||
if cfg.Batch.MaxSize != 10 {
|
||||
t.Errorf("Batch.MaxSize = %d, want 10", cfg.Batch.MaxSize)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadInvalidBatchSizeErrors(t *testing.T) {
|
||||
t.Setenv("CONSUMER_BATCH_MAX_SIZE", "not-a-number")
|
||||
if _, err := Load(); err == nil {
|
||||
t.Fatal("expected error for non-numeric CONSUMER_BATCH_MAX_SIZE, got nil")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
// Package consumer reads normalized-on-write LogRecords back off Redpanda
|
||||
// and batch-writes them into ClickHouse. Offsets are committed only after
|
||||
// a successful ClickHouse write, so a ClickHouse outage causes redelivery
|
||||
// on restart rather than silent data loss (at-least-once, not exactly-once
|
||||
// — Phase 0 doesn't dedupe on the consumer side).
|
||||
package consumer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/segmentio/kafka-go"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/sentry/sentry/ingest/internal/config"
|
||||
logsv1 "github.com/sentry/sentry/proto/sentry/logs/v1"
|
||||
)
|
||||
|
||||
// chWriter is the subset of *clickhousewriter.Writer this package depends
|
||||
// on, kept as an interface so the flush loop is unit-testable without a
|
||||
// real ClickHouse connection.
|
||||
type chWriter interface {
|
||||
WriteBatch(ctx context.Context, records []*logsv1.LogRecord) error
|
||||
}
|
||||
|
||||
// reader is the subset of *kafka.Reader used here, as an interface so the
|
||||
// flush/commit logic can be tested against a fake without a real broker.
|
||||
type reader interface {
|
||||
FetchMessage(ctx context.Context) (kafka.Message, error)
|
||||
CommitMessages(ctx context.Context, msgs ...kafka.Message) error
|
||||
Close() error
|
||||
}
|
||||
|
||||
type Consumer struct {
|
||||
logger *slog.Logger
|
||||
reader reader
|
||||
writer chWriter
|
||||
batchCfg config.BatchConfig
|
||||
}
|
||||
|
||||
func New(logger *slog.Logger, redpandaCfg config.RedpandaConfig, batchCfg config.BatchConfig, w chWriter) *Consumer {
|
||||
r := kafka.NewReader(kafka.ReaderConfig{
|
||||
Brokers: redpandaCfg.Brokers,
|
||||
Topic: redpandaCfg.Topic,
|
||||
GroupID: redpandaCfg.ConsumerGroup,
|
||||
})
|
||||
return &Consumer{logger: logger, reader: r, writer: w, batchCfg: batchCfg}
|
||||
}
|
||||
|
||||
func (c *Consumer) Run(ctx context.Context) error {
|
||||
defer c.reader.Close()
|
||||
|
||||
flushInterval := time.Duration(c.batchCfg.FlushIntervalMS) * time.Millisecond
|
||||
ticker := time.NewTicker(flushInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
msgCh := make(chan kafka.Message)
|
||||
fetchErrCh := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
for {
|
||||
m, err := c.reader.FetchMessage(ctx)
|
||||
if err != nil {
|
||||
fetchErrCh <- err
|
||||
return
|
||||
}
|
||||
select {
|
||||
case msgCh <- m:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
var records []*logsv1.LogRecord
|
||||
var pending []kafka.Message
|
||||
|
||||
flush := func() {
|
||||
if len(records) == 0 {
|
||||
return
|
||||
}
|
||||
if err := c.writer.WriteBatch(ctx, records); err != nil {
|
||||
c.logger.Error("clickhouse batch write failed, offsets not committed, will redeliver",
|
||||
"records", len(records), "error", err)
|
||||
} else if err := c.reader.CommitMessages(ctx, pending...); err != nil {
|
||||
c.logger.Error("committing offsets after clickhouse write", "error", err)
|
||||
} else {
|
||||
c.logger.Debug("batch flushed to clickhouse", "records", len(records))
|
||||
}
|
||||
records = records[:0]
|
||||
pending = pending[:0]
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
flush()
|
||||
return nil
|
||||
case err := <-fetchErrCh:
|
||||
flush()
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
case <-ticker.C:
|
||||
flush()
|
||||
case m := <-msgCh:
|
||||
var rec logsv1.LogRecord
|
||||
if err := proto.Unmarshal(m.Value, &rec); err != nil {
|
||||
c.logger.Warn("skipping unparseable message", "error", err, "offset", m.Offset)
|
||||
if cerr := c.reader.CommitMessages(ctx, m); cerr != nil {
|
||||
c.logger.Error("committing offset for poison message", "error", cerr)
|
||||
}
|
||||
continue
|
||||
}
|
||||
records = append(records, &rec)
|
||||
pending = append(pending, m)
|
||||
if len(records) >= c.batchCfg.MaxSize {
|
||||
flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
package consumer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/segmentio/kafka-go"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/sentry/sentry/ingest/internal/config"
|
||||
logsv1 "github.com/sentry/sentry/proto/sentry/logs/v1"
|
||||
)
|
||||
|
||||
type fakeReader struct {
|
||||
msgs chan kafka.Message
|
||||
|
||||
mu sync.Mutex
|
||||
committed [][]kafka.Message
|
||||
}
|
||||
|
||||
func newFakeReader() *fakeReader {
|
||||
return &fakeReader{msgs: make(chan kafka.Message, 16)}
|
||||
}
|
||||
|
||||
func (f *fakeReader) push(m kafka.Message) { f.msgs <- m }
|
||||
|
||||
func (f *fakeReader) FetchMessage(ctx context.Context) (kafka.Message, error) {
|
||||
select {
|
||||
case m := <-f.msgs:
|
||||
return m, nil
|
||||
case <-ctx.Done():
|
||||
return kafka.Message{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeReader) CommitMessages(_ context.Context, msgs ...kafka.Message) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.committed = append(f.committed, msgs)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeReader) Close() error { return nil }
|
||||
|
||||
func (f *fakeReader) commitCount() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return len(f.committed)
|
||||
}
|
||||
|
||||
type fakeWriter struct {
|
||||
mu sync.Mutex
|
||||
batches [][]*logsv1.LogRecord
|
||||
failNext bool
|
||||
}
|
||||
|
||||
func (f *fakeWriter) WriteBatch(_ context.Context, records []*logsv1.LogRecord) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.failNext {
|
||||
f.failNext = false
|
||||
return errors.New("simulated clickhouse failure")
|
||||
}
|
||||
batch := make([]*logsv1.LogRecord, len(records))
|
||||
copy(batch, records)
|
||||
f.batches = append(f.batches, batch)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeWriter) batchCount() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return len(f.batches)
|
||||
}
|
||||
|
||||
func newTestConsumer(r reader, w chWriter, batchCfg config.BatchConfig) *Consumer {
|
||||
return &Consumer{
|
||||
logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
reader: r,
|
||||
writer: w,
|
||||
batchCfg: batchCfg,
|
||||
}
|
||||
}
|
||||
|
||||
func mustMarshal(t *testing.T, rec *logsv1.LogRecord) []byte {
|
||||
t.Helper()
|
||||
b, err := proto.Marshal(rec)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal: %v", err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func waitFor(t *testing.T, timeout time.Duration, cond func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
if cond() {
|
||||
return
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("condition not met before timeout")
|
||||
}
|
||||
|
||||
func TestConsumerFlushesOnBatchSize(t *testing.T) {
|
||||
fr := newFakeReader()
|
||||
fw := &fakeWriter{}
|
||||
c := newTestConsumer(fr, fw, config.BatchConfig{MaxSize: 2, FlushIntervalMS: 60_000})
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- c.Run(ctx) }()
|
||||
|
||||
fr.push(kafka.Message{Value: mustMarshal(t, &logsv1.LogRecord{Message: "a"})})
|
||||
fr.push(kafka.Message{Value: mustMarshal(t, &logsv1.LogRecord{Message: "b"})})
|
||||
|
||||
waitFor(t, time.Second, func() bool { return fw.batchCount() == 1 })
|
||||
|
||||
fw.mu.Lock()
|
||||
if len(fw.batches[0]) != 2 {
|
||||
t.Fatalf("expected batch of 2 records, got %d", len(fw.batches[0]))
|
||||
}
|
||||
fw.mu.Unlock()
|
||||
|
||||
waitFor(t, time.Second, func() bool { return fr.commitCount() == 1 })
|
||||
}
|
||||
|
||||
func TestConsumerFlushesOnTimeout(t *testing.T) {
|
||||
fr := newFakeReader()
|
||||
fw := &fakeWriter{}
|
||||
c := newTestConsumer(fr, fw, config.BatchConfig{MaxSize: 1000, FlushIntervalMS: 20})
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- c.Run(ctx) }()
|
||||
|
||||
fr.push(kafka.Message{Value: mustMarshal(t, &logsv1.LogRecord{Message: "only-one"})})
|
||||
|
||||
waitFor(t, time.Second, func() bool { return fw.batchCount() == 1 })
|
||||
|
||||
fw.mu.Lock()
|
||||
if len(fw.batches[0]) != 1 {
|
||||
t.Fatalf("expected batch of 1 record, got %d", len(fw.batches[0]))
|
||||
}
|
||||
fw.mu.Unlock()
|
||||
}
|
||||
|
||||
func TestConsumerDoesNotCommitOnWriteFailure(t *testing.T) {
|
||||
fr := newFakeReader()
|
||||
fw := &fakeWriter{failNext: true}
|
||||
c := newTestConsumer(fr, fw, config.BatchConfig{MaxSize: 1, FlushIntervalMS: 60_000})
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- c.Run(ctx) }()
|
||||
|
||||
fr.push(kafka.Message{Value: mustMarshal(t, &logsv1.LogRecord{Message: "will-fail"})})
|
||||
|
||||
// Give the flush a moment to run and fail.
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
if got := fr.commitCount(); got != 0 {
|
||||
t.Fatalf("expected no commits after a failed clickhouse write, got %d", got)
|
||||
}
|
||||
// The batch was attempted even though writer returned an error.
|
||||
if fw.batchCount() != 0 {
|
||||
t.Fatalf("fakeWriter should not record a failed batch, got %d recorded", fw.batchCount())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
// Package grpcserver implements the agent-facing side of ingest: an mTLS
|
||||
// gRPC server accepting LogIngest.PushBatch calls, which it forwards
|
||||
// unchanged (proto-encoded) onto Redpanda. Normalization into the
|
||||
// ClickHouse row shape happens later, on the consumer side.
|
||||
package grpcserver
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
|
||||
"github.com/segmentio/kafka-go"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/credentials"
|
||||
"google.golang.org/grpc/status"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/sentry/sentry/ingest/internal/config"
|
||||
logsv1 "github.com/sentry/sentry/proto/sentry/logs/v1"
|
||||
)
|
||||
|
||||
type Server struct {
|
||||
logsv1.UnimplementedLogIngestServer
|
||||
|
||||
logger *slog.Logger
|
||||
grpcCfg config.GRPCConfig
|
||||
tlsCfg config.TLSConfig
|
||||
producer batchProducer
|
||||
}
|
||||
|
||||
// batchProducer is the subset of *producer.Producer this package depends
|
||||
// on, so tests can substitute a fake without touching Redpanda.
|
||||
type batchProducer interface {
|
||||
WriteBatch(ctx context.Context, msgs []kafka.Message) error
|
||||
}
|
||||
|
||||
func New(logger *slog.Logger, grpcCfg config.GRPCConfig, tlsCfg config.TLSConfig, p batchProducer) *Server {
|
||||
return &Server{logger: logger, grpcCfg: grpcCfg, tlsCfg: tlsCfg, producer: p}
|
||||
}
|
||||
|
||||
// Run blocks serving gRPC until ctx is canceled, then gracefully stops.
|
||||
func (s *Server) Run(ctx context.Context) error {
|
||||
tlsConf, err := loadServerTLSConfig(s.tlsCfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("loading TLS config: %w", err)
|
||||
}
|
||||
|
||||
lis, err := net.Listen("tcp", s.grpcCfg.ListenAddr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listening on %s: %w", s.grpcCfg.ListenAddr, err)
|
||||
}
|
||||
|
||||
grpcSrv := grpc.NewServer(grpc.Creds(credentials.NewTLS(tlsConf)))
|
||||
logsv1.RegisterLogIngestServer(grpcSrv, s)
|
||||
|
||||
s.logger.Info("gRPC server listening", "addr", s.grpcCfg.ListenAddr)
|
||||
|
||||
errCh := make(chan error, 1)
|
||||
go func() { errCh <- grpcSrv.Serve(lis) }()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
grpcSrv.GracefulStop()
|
||||
return nil
|
||||
case err := <-errCh:
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) PushBatch(ctx context.Context, req *logsv1.PushBatchRequest) (*logsv1.PushBatchResponse, error) {
|
||||
if len(req.GetRecords()) == 0 {
|
||||
return &logsv1.PushBatchResponse{Accepted: 0}, nil
|
||||
}
|
||||
|
||||
msgs := make([]kafka.Message, 0, len(req.GetRecords()))
|
||||
for _, rec := range req.GetRecords() {
|
||||
val, err := proto.Marshal(rec)
|
||||
if err != nil {
|
||||
return nil, status.Errorf(codes.InvalidArgument, "marshaling record: %v", err)
|
||||
}
|
||||
msgs = append(msgs, kafka.Message{
|
||||
Key: []byte(rec.GetHost()),
|
||||
Value: val,
|
||||
})
|
||||
}
|
||||
|
||||
if err := s.producer.WriteBatch(ctx, msgs); err != nil {
|
||||
s.logger.Error("failed to write batch to redpanda", "batch_id", req.GetBatchId(), "error", err)
|
||||
return nil, status.Errorf(codes.Unavailable, "writing to transport: %v", err)
|
||||
}
|
||||
|
||||
s.logger.Debug("batch produced to redpanda", "batch_id", req.GetBatchId(), "records", len(req.GetRecords()))
|
||||
return &logsv1.PushBatchResponse{Accepted: uint32(len(req.GetRecords()))}, nil
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package grpcserver
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/sentry/sentry/ingest/internal/config"
|
||||
)
|
||||
|
||||
// loadServerTLSConfig builds the mTLS server config: ingest's own
|
||||
// certificate, plus the CA used to verify agent client certificates.
|
||||
// Agents are never accepted without a client cert signed by this CA.
|
||||
func loadServerTLSConfig(cfg config.TLSConfig) (*tls.Config, error) {
|
||||
cert, err := tls.LoadX509KeyPair(cfg.CertFile, cfg.KeyFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("loading server cert/key: %w", err)
|
||||
}
|
||||
|
||||
caPEM, err := os.ReadFile(cfg.ClientCAFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading client CA file: %w", err)
|
||||
}
|
||||
caPool := x509.NewCertPool()
|
||||
if !caPool.AppendCertsFromPEM(caPEM) {
|
||||
return nil, fmt.Errorf("no valid certificates found in client CA file %s", cfg.ClientCAFile)
|
||||
}
|
||||
|
||||
return &tls.Config{
|
||||
Certificates: []tls.Certificate{cert},
|
||||
ClientAuth: tls.RequireAndVerifyClientCert,
|
||||
ClientCAs: caPool,
|
||||
MinVersion: tls.VersionTLS12,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
// Package normalize maps the wire-format LogRecord (as agents send it)
|
||||
// into the ClickHouse row shape defined in /storage. This is the "OTel-log-
|
||||
// like schema" normalization step called for in the ingest design — Phase
|
||||
// 0 keeps it to the minimal column set; full OTel field mapping (separate
|
||||
// SeverityNumber/SeverityText, resource attributes, etc.) is deferred, see
|
||||
// the open questions in /docs/architecture.md.
|
||||
package normalize
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
logsv1 "github.com/sentry/sentry/proto/sentry/logs/v1"
|
||||
)
|
||||
|
||||
type Row struct {
|
||||
Timestamp time.Time
|
||||
Host string
|
||||
Service string
|
||||
Severity string
|
||||
Message string
|
||||
Attributes map[string]string
|
||||
}
|
||||
|
||||
func ToRow(rec *logsv1.LogRecord) Row {
|
||||
attrs := rec.GetAttributes()
|
||||
if attrs == nil {
|
||||
attrs = map[string]string{}
|
||||
}
|
||||
return Row{
|
||||
Timestamp: time.Unix(0, rec.GetTimestampUnixNano()).UTC(),
|
||||
Host: rec.GetHost(),
|
||||
Service: rec.GetService(),
|
||||
Severity: severityText(rec.GetSeverity()),
|
||||
Message: rec.GetMessage(),
|
||||
Attributes: attrs,
|
||||
}
|
||||
}
|
||||
|
||||
// severityText maps the proto Severity enum to short OTel-style severity
|
||||
// names, stored as the `severity` column's value.
|
||||
func severityText(sev logsv1.Severity) string {
|
||||
switch sev {
|
||||
case logsv1.Severity_SEVERITY_TRACE:
|
||||
return "TRACE"
|
||||
case logsv1.Severity_SEVERITY_DEBUG:
|
||||
return "DEBUG"
|
||||
case logsv1.Severity_SEVERITY_INFO:
|
||||
return "INFO"
|
||||
case logsv1.Severity_SEVERITY_WARN:
|
||||
return "WARN"
|
||||
case logsv1.Severity_SEVERITY_ERROR:
|
||||
return "ERROR"
|
||||
case logsv1.Severity_SEVERITY_FATAL:
|
||||
return "FATAL"
|
||||
default:
|
||||
return "UNSPECIFIED"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package normalize
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
logsv1 "github.com/sentry/sentry/proto/sentry/logs/v1"
|
||||
)
|
||||
|
||||
func TestToRowMapsFieldsAndSeverity(t *testing.T) {
|
||||
rec := &logsv1.LogRecord{
|
||||
TimestampUnixNano: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC).UnixNano(),
|
||||
Host: "host-1",
|
||||
Service: "svc-a",
|
||||
Severity: logsv1.Severity_SEVERITY_ERROR,
|
||||
Message: "boom",
|
||||
Attributes: map[string]string{"k": "v"},
|
||||
}
|
||||
|
||||
row := ToRow(rec)
|
||||
|
||||
if row.Host != "host-1" || row.Service != "svc-a" || row.Message != "boom" {
|
||||
t.Fatalf("unexpected row: %+v", row)
|
||||
}
|
||||
if row.Severity != "ERROR" {
|
||||
t.Fatalf("expected severity ERROR, got %s", row.Severity)
|
||||
}
|
||||
if row.Attributes["k"] != "v" {
|
||||
t.Fatalf("expected attribute k=v, got %+v", row.Attributes)
|
||||
}
|
||||
if !row.Timestamp.Equal(time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)) {
|
||||
t.Fatalf("unexpected timestamp: %v", row.Timestamp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestToRowNilAttributesBecomesEmptyMap(t *testing.T) {
|
||||
rec := &logsv1.LogRecord{Host: "h", Service: "s", Message: "m"}
|
||||
row := ToRow(rec)
|
||||
if row.Attributes == nil {
|
||||
t.Fatal("expected non-nil empty map, got nil")
|
||||
}
|
||||
if len(row.Attributes) != 0 {
|
||||
t.Fatalf("expected empty map, got %+v", row.Attributes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeverityTextCoversAllEnumValues(t *testing.T) {
|
||||
cases := map[logsv1.Severity]string{
|
||||
logsv1.Severity_SEVERITY_UNSPECIFIED: "UNSPECIFIED",
|
||||
logsv1.Severity_SEVERITY_TRACE: "TRACE",
|
||||
logsv1.Severity_SEVERITY_DEBUG: "DEBUG",
|
||||
logsv1.Severity_SEVERITY_INFO: "INFO",
|
||||
logsv1.Severity_SEVERITY_WARN: "WARN",
|
||||
logsv1.Severity_SEVERITY_ERROR: "ERROR",
|
||||
logsv1.Severity_SEVERITY_FATAL: "FATAL",
|
||||
}
|
||||
for sev, want := range cases {
|
||||
if got := severityText(sev); got != want {
|
||||
t.Errorf("severityText(%v) = %q, want %q", sev, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeverityTextUnknownValueFallsBackToUnspecified(t *testing.T) {
|
||||
if got := severityText(logsv1.Severity(99)); got != "UNSPECIFIED" {
|
||||
t.Fatalf("expected UNSPECIFIED for unknown severity, got %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
// Package producer wraps the Redpanda (Kafka API) producer used by the
|
||||
// gRPC front end to forward agent-submitted batches onto the transport
|
||||
// layer, unchanged. OTel-log-shape normalization happens later, on the
|
||||
// consumer side — see internal/normalize.
|
||||
package producer
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/segmentio/kafka-go"
|
||||
|
||||
"github.com/sentry/sentry/ingest/internal/config"
|
||||
)
|
||||
|
||||
type Producer struct {
|
||||
writer *kafka.Writer
|
||||
}
|
||||
|
||||
func New(cfg config.RedpandaConfig) *Producer {
|
||||
return &Producer{
|
||||
writer: &kafka.Writer{
|
||||
Addr: kafka.TCP(cfg.Brokers...),
|
||||
Topic: cfg.Topic,
|
||||
// Partition by host so a single host's records stay in
|
||||
// relative order within a partition.
|
||||
Balancer: &kafka.Hash{},
|
||||
RequiredAcks: kafka.RequireOne,
|
||||
AllowAutoTopicCreation: false, // topics are provisioned explicitly, see /transport
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Producer) Close() error {
|
||||
return p.writer.Close()
|
||||
}
|
||||
|
||||
// WriteBatch writes all messages in one call. kafka-go's WriteMessages
|
||||
// either succeeds for the whole batch or returns an error, which matches
|
||||
// the PushBatch RPC's all-or-nothing contract for Phase 0.
|
||||
func (p *Producer) WriteBatch(ctx context.Context, msgs []kafka.Message) error {
|
||||
return p.writer.WriteMessages(ctx, msgs...)
|
||||
}
|
||||
Reference in New Issue
Block a user