Build per-tenant ClickHouse write-routing for ingest (Tantivy still deferred)
ingest tags every record with a tenant_id Kafka header (built previously), but nothing consumed it to actually route the write. This closes that for ClickHouse: enterprise/cmd/enterprise-ingest (a second binary, mirroring enterprise-api) reuses ingest/consumer's own flush loop unchanged, with enterprise/internal/chwriter.Registry -- a per-tenant clickhousewriter.Writer registry -- swapped in as the writer. A batch pulled from the single shared Redpanda topic can mix records from many tenants, so WriteBatch groups by TenantID and dispatches each group to its own tenant's connection, fail- closed on an empty or unrecognized tenant_id. ingest/consumer and ingest/clickhousewriter move out of internal/ (same reason api/internal/* moved earlier this phase: enterprise/ can't import anything under another module's internal/). Their New() constructors now take small local Config structs instead of ingest/internal/config types, so enterprise/ doesn't need that import either. Building this surfaced a real bug: tenantprovision.ProvisionClickHouse only granted SELECT on a tenant's ClickHouse user, correct for chrunner's read-only use but not enough for chwriter reusing the same credential to write -- every real per-tenant write would have failed closed with a permission error. Fixed by widening the grant to SELECT, INSERT; no cross-tenant boundary is crossed by also allowing INSERT within a tenant's own database. Helm gates enterprise-ingest's Deployment on the same ingest.requireTenantCredential flag that already gates tag validation -- write-routing is meaningless without tagging already being required, so they're one decision, not two. docker-compose.yml's version is a disclosed, weaker approximation: it can't achieve Helm's genuine -mode=server/-mode=consumer split, so with the enterprise profile active both ingest and enterprise-ingest independently consume every message via different consumer groups -- harmless duplication for local verification only. Not built: Tantivy's independent Redpanda consumer (search/src/consumer.rs) still doesn't read the tenant_id header at all -- every record still lands in the one shared index regardless of tenant. Not run: the live-ClickHouse- gated tests (chwriter's cross-tenant routing test, tenantprovision's INSERT regression test) -- no Docker/database access in this environment; they're correct Go that has never executed, disclosed as such in docs/security/ threat-model.md and docs/phase-4-runbook.md §14.
This commit is contained in:
@@ -1,60 +0,0 @@
|
||||
// Package clickhousewriter batch-inserts normalized log rows into
|
||||
// ClickHouse using the native protocol driver's batch API.
|
||||
package clickhousewriter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2"
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
|
||||
"github.com/sentry/sentry/ingest/internal/config"
|
||||
"github.com/sentry/sentry/ingest/internal/normalize"
|
||||
logsv1 "github.com/sentry/sentry/proto/sentry/logs/v1"
|
||||
)
|
||||
|
||||
type Writer struct {
|
||||
conn driver.Conn
|
||||
}
|
||||
|
||||
func New(ctx context.Context, cfg config.ClickHouseConfig) (*Writer, error) {
|
||||
conn, err := clickhouse.Open(&clickhouse.Options{
|
||||
Addr: []string{cfg.Addr},
|
||||
Auth: clickhouse.Auth{
|
||||
Database: cfg.Database,
|
||||
Username: cfg.Username,
|
||||
Password: cfg.Password,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("opening clickhouse connection: %w", err)
|
||||
}
|
||||
if err := conn.Ping(ctx); err != nil {
|
||||
return nil, fmt.Errorf("pinging clickhouse: %w", err)
|
||||
}
|
||||
return &Writer{conn: conn}, nil
|
||||
}
|
||||
|
||||
func (w *Writer) Close() error {
|
||||
return w.conn.Close()
|
||||
}
|
||||
|
||||
func (w *Writer) WriteBatch(ctx context.Context, records []*logsv1.LogRecord) error {
|
||||
batch, err := w.conn.PrepareBatch(ctx, "INSERT INTO logs (timestamp, host, service, severity, message, attributes, record_id)")
|
||||
if err != nil {
|
||||
return fmt.Errorf("preparing batch: %w", err)
|
||||
}
|
||||
|
||||
for _, rec := range records {
|
||||
row := normalize.ToRow(rec)
|
||||
if err := batch.Append(row.Timestamp, row.Host, row.Service, row.Severity, row.Message, row.Attributes, row.RecordID); err != nil {
|
||||
return fmt.Errorf("appending row to batch: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := batch.Send(); err != nil {
|
||||
return fmt.Errorf("sending batch: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,124 +0,0 @@
|
||||
// Package consumer reads normalized-on-write LogRecords back off Redpanda
|
||||
// and batch-writes them into ClickHouse. Offsets are committed only after
|
||||
// a successful ClickHouse write, so a ClickHouse outage causes redelivery
|
||||
// on restart rather than silent data loss (at-least-once, not exactly-once
|
||||
// — Phase 0 doesn't dedupe on the consumer side).
|
||||
package consumer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/segmentio/kafka-go"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/sentry/sentry/ingest/internal/config"
|
||||
logsv1 "github.com/sentry/sentry/proto/sentry/logs/v1"
|
||||
)
|
||||
|
||||
// chWriter is the subset of *clickhousewriter.Writer this package depends
|
||||
// on, kept as an interface so the flush loop is unit-testable without a
|
||||
// real ClickHouse connection.
|
||||
type chWriter interface {
|
||||
WriteBatch(ctx context.Context, records []*logsv1.LogRecord) error
|
||||
}
|
||||
|
||||
// reader is the subset of *kafka.Reader used here, as an interface so the
|
||||
// flush/commit logic can be tested against a fake without a real broker.
|
||||
type reader interface {
|
||||
FetchMessage(ctx context.Context) (kafka.Message, error)
|
||||
CommitMessages(ctx context.Context, msgs ...kafka.Message) error
|
||||
Close() error
|
||||
}
|
||||
|
||||
type Consumer struct {
|
||||
logger *slog.Logger
|
||||
reader reader
|
||||
writer chWriter
|
||||
batchCfg config.BatchConfig
|
||||
}
|
||||
|
||||
func New(logger *slog.Logger, redpandaCfg config.RedpandaConfig, batchCfg config.BatchConfig, w chWriter) *Consumer {
|
||||
r := kafka.NewReader(kafka.ReaderConfig{
|
||||
Brokers: redpandaCfg.Brokers,
|
||||
Topic: redpandaCfg.Topic,
|
||||
GroupID: redpandaCfg.ConsumerGroup,
|
||||
})
|
||||
return &Consumer{logger: logger, reader: r, writer: w, batchCfg: batchCfg}
|
||||
}
|
||||
|
||||
func (c *Consumer) Run(ctx context.Context) error {
|
||||
defer c.reader.Close()
|
||||
|
||||
flushInterval := time.Duration(c.batchCfg.FlushIntervalMS) * time.Millisecond
|
||||
ticker := time.NewTicker(flushInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
msgCh := make(chan kafka.Message)
|
||||
fetchErrCh := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
for {
|
||||
m, err := c.reader.FetchMessage(ctx)
|
||||
if err != nil {
|
||||
fetchErrCh <- err
|
||||
return
|
||||
}
|
||||
select {
|
||||
case msgCh <- m:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
var records []*logsv1.LogRecord
|
||||
var pending []kafka.Message
|
||||
|
||||
flush := func() {
|
||||
if len(records) == 0 {
|
||||
return
|
||||
}
|
||||
if err := c.writer.WriteBatch(ctx, records); err != nil {
|
||||
c.logger.Error("clickhouse batch write failed, offsets not committed, will redeliver",
|
||||
"records", len(records), "error", err)
|
||||
} else if err := c.reader.CommitMessages(ctx, pending...); err != nil {
|
||||
c.logger.Error("committing offsets after clickhouse write", "error", err)
|
||||
} else {
|
||||
c.logger.Debug("batch flushed to clickhouse", "records", len(records))
|
||||
}
|
||||
records = records[:0]
|
||||
pending = pending[:0]
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
flush()
|
||||
return nil
|
||||
case err := <-fetchErrCh:
|
||||
flush()
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
case <-ticker.C:
|
||||
flush()
|
||||
case m := <-msgCh:
|
||||
var rec logsv1.LogRecord
|
||||
if err := proto.Unmarshal(m.Value, &rec); err != nil {
|
||||
c.logger.Warn("skipping unparseable message", "error", err, "offset", m.Offset)
|
||||
if cerr := c.reader.CommitMessages(ctx, m); cerr != nil {
|
||||
c.logger.Error("committing offset for poison message", "error", cerr)
|
||||
}
|
||||
continue
|
||||
}
|
||||
records = append(records, &rec)
|
||||
pending = append(pending, m)
|
||||
if len(records) >= c.batchCfg.MaxSize {
|
||||
flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,178 +0,0 @@
|
||||
package consumer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/segmentio/kafka-go"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/sentry/sentry/ingest/internal/config"
|
||||
logsv1 "github.com/sentry/sentry/proto/sentry/logs/v1"
|
||||
)
|
||||
|
||||
type fakeReader struct {
|
||||
msgs chan kafka.Message
|
||||
|
||||
mu sync.Mutex
|
||||
committed [][]kafka.Message
|
||||
}
|
||||
|
||||
func newFakeReader() *fakeReader {
|
||||
return &fakeReader{msgs: make(chan kafka.Message, 16)}
|
||||
}
|
||||
|
||||
func (f *fakeReader) push(m kafka.Message) { f.msgs <- m }
|
||||
|
||||
func (f *fakeReader) FetchMessage(ctx context.Context) (kafka.Message, error) {
|
||||
select {
|
||||
case m := <-f.msgs:
|
||||
return m, nil
|
||||
case <-ctx.Done():
|
||||
return kafka.Message{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeReader) CommitMessages(_ context.Context, msgs ...kafka.Message) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.committed = append(f.committed, msgs)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeReader) Close() error { return nil }
|
||||
|
||||
func (f *fakeReader) commitCount() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return len(f.committed)
|
||||
}
|
||||
|
||||
type fakeWriter struct {
|
||||
mu sync.Mutex
|
||||
batches [][]*logsv1.LogRecord
|
||||
failNext bool
|
||||
}
|
||||
|
||||
func (f *fakeWriter) WriteBatch(_ context.Context, records []*logsv1.LogRecord) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.failNext {
|
||||
f.failNext = false
|
||||
return errors.New("simulated clickhouse failure")
|
||||
}
|
||||
batch := make([]*logsv1.LogRecord, len(records))
|
||||
copy(batch, records)
|
||||
f.batches = append(f.batches, batch)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeWriter) batchCount() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return len(f.batches)
|
||||
}
|
||||
|
||||
func newTestConsumer(r reader, w chWriter, batchCfg config.BatchConfig) *Consumer {
|
||||
return &Consumer{
|
||||
logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
reader: r,
|
||||
writer: w,
|
||||
batchCfg: batchCfg,
|
||||
}
|
||||
}
|
||||
|
||||
func mustMarshal(t *testing.T, rec *logsv1.LogRecord) []byte {
|
||||
t.Helper()
|
||||
b, err := proto.Marshal(rec)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal: %v", err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func waitFor(t *testing.T, timeout time.Duration, cond func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
if cond() {
|
||||
return
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("condition not met before timeout")
|
||||
}
|
||||
|
||||
func TestConsumerFlushesOnBatchSize(t *testing.T) {
|
||||
fr := newFakeReader()
|
||||
fw := &fakeWriter{}
|
||||
c := newTestConsumer(fr, fw, config.BatchConfig{MaxSize: 2, FlushIntervalMS: 60_000})
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- c.Run(ctx) }()
|
||||
|
||||
fr.push(kafka.Message{Value: mustMarshal(t, &logsv1.LogRecord{Message: "a"})})
|
||||
fr.push(kafka.Message{Value: mustMarshal(t, &logsv1.LogRecord{Message: "b"})})
|
||||
|
||||
waitFor(t, time.Second, func() bool { return fw.batchCount() == 1 })
|
||||
|
||||
fw.mu.Lock()
|
||||
if len(fw.batches[0]) != 2 {
|
||||
t.Fatalf("expected batch of 2 records, got %d", len(fw.batches[0]))
|
||||
}
|
||||
fw.mu.Unlock()
|
||||
|
||||
waitFor(t, time.Second, func() bool { return fr.commitCount() == 1 })
|
||||
}
|
||||
|
||||
func TestConsumerFlushesOnTimeout(t *testing.T) {
|
||||
fr := newFakeReader()
|
||||
fw := &fakeWriter{}
|
||||
c := newTestConsumer(fr, fw, config.BatchConfig{MaxSize: 1000, FlushIntervalMS: 20})
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- c.Run(ctx) }()
|
||||
|
||||
fr.push(kafka.Message{Value: mustMarshal(t, &logsv1.LogRecord{Message: "only-one"})})
|
||||
|
||||
waitFor(t, time.Second, func() bool { return fw.batchCount() == 1 })
|
||||
|
||||
fw.mu.Lock()
|
||||
if len(fw.batches[0]) != 1 {
|
||||
t.Fatalf("expected batch of 1 record, got %d", len(fw.batches[0]))
|
||||
}
|
||||
fw.mu.Unlock()
|
||||
}
|
||||
|
||||
func TestConsumerDoesNotCommitOnWriteFailure(t *testing.T) {
|
||||
fr := newFakeReader()
|
||||
fw := &fakeWriter{failNext: true}
|
||||
c := newTestConsumer(fr, fw, config.BatchConfig{MaxSize: 1, FlushIntervalMS: 60_000})
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- c.Run(ctx) }()
|
||||
|
||||
fr.push(kafka.Message{Value: mustMarshal(t, &logsv1.LogRecord{Message: "will-fail"})})
|
||||
|
||||
// Give the flush a moment to run and fail.
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
if got := fr.commitCount(); got != 0 {
|
||||
t.Fatalf("expected no commits after a failed clickhouse write, got %d", got)
|
||||
}
|
||||
// The batch was attempted even though writer returned an error.
|
||||
if fw.batchCount() != 0 {
|
||||
t.Fatalf("fakeWriter should not record a failed batch, got %d recorded", fw.batchCount())
|
||||
}
|
||||
}
|
||||
@@ -43,9 +43,11 @@ import (
|
||||
)
|
||||
|
||||
// TenantIDHeaderKey is the Kafka message header a resolved tenant ID is
|
||||
// attached under -- exported so internal/consumer (or a future per-
|
||||
// tenant write-routing consumer) can read it back by the same name
|
||||
// without duplicating the literal.
|
||||
// attached under. ingest/consumer.TenantIDHeaderKey names the identical
|
||||
// literal on the read side -- duplicated rather than imported (this
|
||||
// package is the agent-facing producer side; consumer is a different
|
||||
// concern, and importing across them for one string constant isn't
|
||||
// worth the coupling), so a change here must be mirrored there.
|
||||
const TenantIDHeaderKey = "tenant_id"
|
||||
|
||||
type Server struct {
|
||||
|
||||
Reference in New Issue
Block a user