jcoffey-dev
1de77b969f
ingest tags every record with a tenant_id Kafka header (built previously), but nothing consumed it to actually route the write. This closes that for ClickHouse: enterprise/cmd/enterprise-ingest (a second binary, mirroring enterprise-api) reuses ingest/consumer's own flush loop unchanged, with enterprise/internal/chwriter.Registry -- a per-tenant clickhousewriter.Writer registry -- swapped in as the writer. A batch pulled from the single shared Redpanda topic can mix records from many tenants, so WriteBatch groups by TenantID and dispatches each group to its own tenant's connection, fail- closed on an empty or unrecognized tenant_id. ingest/consumer and ingest/clickhousewriter move out of internal/ (same reason api/internal/* moved earlier this phase: enterprise/ can't import anything under another module's internal/). Their New() constructors now take small local Config structs instead of ingest/internal/config types, so enterprise/ doesn't need that import either. Building this surfaced a real bug: tenantprovision.ProvisionClickHouse only granted SELECT on a tenant's ClickHouse user, correct for chrunner's read-only use but not enough for chwriter reusing the same credential to write -- every real per-tenant write would have failed closed with a permission error. Fixed by widening the grant to SELECT, INSERT; no cross-tenant boundary is crossed by also allowing INSERT within a tenant's own database. Helm gates enterprise-ingest's Deployment on the same ingest.requireTenantCredential flag that already gates tag validation -- write-routing is meaningless without tagging already being required, so they're one decision, not two. docker-compose.yml's version is a disclosed, weaker approximation: it can't achieve Helm's genuine -mode=server/-mode=consumer split, so with the enterprise profile active both ingest and enterprise-ingest independently consume every message via different consumer groups -- harmless duplication for local verification only. Not built: Tantivy's independent Redpanda consumer (search/src/consumer.rs) still doesn't read the tenant_id header at all -- every record still lands in the one shared index regardless of tenant. Not run: the live-ClickHouse- gated tests (chwriter's cross-tenant routing test, tenantprovision's INSERT regression test) -- no Docker/database access in this environment; they're correct Go that has never executed, disclosed as such in docs/security/ threat-model.md and docs/phase-4-runbook.md §14.
82 lines
2.6 KiB
Go
82 lines
2.6 KiB
Go
// Package clickhousewriter batch-inserts normalized log rows into
|
|
// ClickHouse using the native protocol driver's batch API. Moved out of
|
|
// internal/ (was ingest/internal/clickhousewriter) once
|
|
// enterprise/internal/chwriter needed to construct one *Writer per
|
|
// tenant -- same reasoning api/internal/dashboards and friends moved out
|
|
// of internal/ earlier in Phase 4: Go's compiler-enforced internal/
|
|
// visibility blocks a separate module (enterprise/) from importing
|
|
// anything under ingest/internal/..., regardless of what the AGPL/
|
|
// commercial licensing boundary itself would otherwise allow.
|
|
package clickhousewriter
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
|
|
"github.com/ClickHouse/clickhouse-go/v2"
|
|
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
|
|
|
"github.com/sentry/sentry/ingest/internal/normalize"
|
|
logsv1 "github.com/sentry/sentry/proto/sentry/logs/v1"
|
|
)
|
|
|
|
// Config is deliberately a local type, not ingest/internal/config.
|
|
// ClickHouseConfig -- this package needs to be importable from
|
|
// enterprise/ (see the package doc comment), and internal/config must
|
|
// stay internal (nothing outside ingest/ needs its other fields, e.g.
|
|
// TLSConfig/GRPCConfig, and moving the whole package out just for this
|
|
// one struct would be a wider hole than necessary). Same "narrow local
|
|
// type, not the storage/config type itself" precedent as
|
|
// enterprise/internal/chrunner.DataSource.
|
|
type Config struct {
|
|
Addr string
|
|
Database string
|
|
Username string
|
|
Password string
|
|
}
|
|
|
|
type Writer struct {
|
|
conn driver.Conn
|
|
}
|
|
|
|
func New(ctx context.Context, cfg Config) (*Writer, error) {
|
|
conn, err := clickhouse.Open(&clickhouse.Options{
|
|
Addr: []string{cfg.Addr},
|
|
Auth: clickhouse.Auth{
|
|
Database: cfg.Database,
|
|
Username: cfg.Username,
|
|
Password: cfg.Password,
|
|
},
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("opening clickhouse connection: %w", err)
|
|
}
|
|
if err := conn.Ping(ctx); err != nil {
|
|
return nil, fmt.Errorf("pinging clickhouse: %w", err)
|
|
}
|
|
return &Writer{conn: conn}, nil
|
|
}
|
|
|
|
func (w *Writer) Close() error {
|
|
return w.conn.Close()
|
|
}
|
|
|
|
func (w *Writer) WriteBatch(ctx context.Context, records []*logsv1.LogRecord) error {
|
|
batch, err := w.conn.PrepareBatch(ctx, "INSERT INTO logs (timestamp, host, service, severity, message, attributes, record_id)")
|
|
if err != nil {
|
|
return fmt.Errorf("preparing batch: %w", err)
|
|
}
|
|
|
|
for _, rec := range records {
|
|
row := normalize.ToRow(rec)
|
|
if err := batch.Append(row.Timestamp, row.Host, row.Service, row.Severity, row.Message, row.Attributes, row.RecordID); err != nil {
|
|
return fmt.Errorf("appending row to batch: %w", err)
|
|
}
|
|
}
|
|
|
|
if err := batch.Send(); err != nil {
|
|
return fmt.Errorf("sending batch: %w", err)
|
|
}
|
|
return nil
|
|
}
|