Scaffold Phase 0: agent -> Redpanda -> ingest -> ClickHouse -> api -> web

End-to-end log pipeline for Linux hosts, per /docs/architecture.md:

- proto: shared gRPC contract (agent <-> ingest), Go bindings checked in
- agent: Rust, musl-targeted, journald/file sourcing, RFC5424 parser,
  mTLS gRPC client, no required config for the common case
- ingest: Go, single binary with --mode server|consumer|all; gRPC front
  end forwards to Redpanda unchanged, consumer normalizes and
  batch-writes to ClickHouse with at-least-once delivery
- storage: ClickHouse schema + a plain SQL-file migration runner
- api: minimal SELECT-only query endpoint, plain REST (not gRPC+gateway
  yet -- see api/README.md)
- web: SvelteKit static SPA, one query page
- transport: Redpanda compose + topic provisioning
- cli: sentryctl ping stub
- hack/dev-certs: throwaway CA + cert generation for local mTLS
- root docker-compose.yml + docs/phase-0-runbook.md tie it together

Not yet run end-to-end against real Docker/ClickHouse/Redpanda -- see the
runbook's caveats section before relying on this working as-is.

ingest/internal/consumer/consumer.go

@@ -0,0 +1,124 @@
+// Package consumer reads normalized-on-write LogRecords back off Redpanda
+// and batch-writes them into ClickHouse. Offsets are committed only after
+// a successful ClickHouse write, so a ClickHouse outage causes redelivery
+// on restart rather than silent data loss (at-least-once, not exactly-once
+// — Phase 0 doesn't dedupe on the consumer side).
+package consumer
+
+import (
+	"context"
+	"log/slog"
+	"time"
+
+	"github.com/segmentio/kafka-go"
+	"google.golang.org/protobuf/proto"
+
+	"github.com/sentry/sentry/ingest/internal/config"
+	logsv1 "github.com/sentry/sentry/proto/sentry/logs/v1"
+)
+
+// chWriter is the subset of *clickhousewriter.Writer this package depends
+// on, kept as an interface so the flush loop is unit-testable without a
+// real ClickHouse connection.
+type chWriter interface {
+	WriteBatch(ctx context.Context, records []*logsv1.LogRecord) error
+}
+
+// reader is the subset of *kafka.Reader used here, as an interface so the
+// flush/commit logic can be tested against a fake without a real broker.
+type reader interface {
+	FetchMessage(ctx context.Context) (kafka.Message, error)
+	CommitMessages(ctx context.Context, msgs ...kafka.Message) error
+	Close() error
+}
+
+type Consumer struct {
+	logger   *slog.Logger
+	reader   reader
+	writer   chWriter
+	batchCfg config.BatchConfig
+}
+
+func New(logger *slog.Logger, redpandaCfg config.RedpandaConfig, batchCfg config.BatchConfig, w chWriter) *Consumer {
+	r := kafka.NewReader(kafka.ReaderConfig{
+		Brokers: redpandaCfg.Brokers,
+		Topic:   redpandaCfg.Topic,
+		GroupID: redpandaCfg.ConsumerGroup,
+	})
+	return &Consumer{logger: logger, reader: r, writer: w, batchCfg: batchCfg}
+}
+
+func (c *Consumer) Run(ctx context.Context) error {
+	defer c.reader.Close()
+
+	flushInterval := time.Duration(c.batchCfg.FlushIntervalMS) * time.Millisecond
+	ticker := time.NewTicker(flushInterval)
+	defer ticker.Stop()
+
+	msgCh := make(chan kafka.Message)
+	fetchErrCh := make(chan error, 1)
+
+	go func() {
+		for {
+			m, err := c.reader.FetchMessage(ctx)
+			if err != nil {
+				fetchErrCh <- err
+				return
+			}
+			select {
+			case msgCh <- m:
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	var records []*logsv1.LogRecord
+	var pending []kafka.Message
+
+	flush := func() {
+		if len(records) == 0 {
+			return
+		}
+		if err := c.writer.WriteBatch(ctx, records); err != nil {
+			c.logger.Error("clickhouse batch write failed, offsets not committed, will redeliver",
+				"records", len(records), "error", err)
+		} else if err := c.reader.CommitMessages(ctx, pending...); err != nil {
+			c.logger.Error("committing offsets after clickhouse write", "error", err)
+		} else {
+			c.logger.Debug("batch flushed to clickhouse", "records", len(records))
+		}
+		records = records[:0]
+		pending = pending[:0]
+	}
+
+	for {
+		select {
+		case <-ctx.Done():
+			flush()
+			return nil
+		case err := <-fetchErrCh:
+			flush()
+			if ctx.Err() != nil {
+				return nil
+			}
+			return err
+		case <-ticker.C:
+			flush()
+		case m := <-msgCh:
+			var rec logsv1.LogRecord
+			if err := proto.Unmarshal(m.Value, &rec); err != nil {
+				c.logger.Warn("skipping unparseable message", "error", err, "offset", m.Offset)
+				if cerr := c.reader.CommitMessages(ctx, m); cerr != nil {
+					c.logger.Error("committing offset for poison message", "error", cerr)
+				}
+				continue
+			}
+			records = append(records, &rec)
+			pending = append(pending, m)
+			if len(records) >= c.batchCfg.MaxSize {
+				flush()
+			}
+		}
+	}
+}