Build per-tenant ClickHouse write-routing for ingest (Tantivy still deferred)
ingest tags every record with a tenant_id Kafka header (built previously), but nothing consumed it to actually route the write. This closes that for ClickHouse: enterprise/cmd/enterprise-ingest (a second binary, mirroring enterprise-api) reuses ingest/consumer's own flush loop unchanged, with enterprise/internal/chwriter.Registry -- a per-tenant clickhousewriter.Writer registry -- swapped in as the writer. A batch pulled from the single shared Redpanda topic can mix records from many tenants, so WriteBatch groups by TenantID and dispatches each group to its own tenant's connection, fail- closed on an empty or unrecognized tenant_id. ingest/consumer and ingest/clickhousewriter move out of internal/ (same reason api/internal/* moved earlier this phase: enterprise/ can't import anything under another module's internal/). Their New() constructors now take small local Config structs instead of ingest/internal/config types, so enterprise/ doesn't need that import either. Building this surfaced a real bug: tenantprovision.ProvisionClickHouse only granted SELECT on a tenant's ClickHouse user, correct for chrunner's read-only use but not enough for chwriter reusing the same credential to write -- every real per-tenant write would have failed closed with a permission error. Fixed by widening the grant to SELECT, INSERT; no cross-tenant boundary is crossed by also allowing INSERT within a tenant's own database. Helm gates enterprise-ingest's Deployment on the same ingest.requireTenantCredential flag that already gates tag validation -- write-routing is meaningless without tagging already being required, so they're one decision, not two. docker-compose.yml's version is a disclosed, weaker approximation: it can't achieve Helm's genuine -mode=server/-mode=consumer split, so with the enterprise profile active both ingest and enterprise-ingest independently consume every message via different consumer groups -- harmless duplication for local verification only. Not built: Tantivy's independent Redpanda consumer (search/src/consumer.rs) still doesn't read the tenant_id header at all -- every record still lands in the one shared index regardless of tenant. Not run: the live-ClickHouse- gated tests (chwriter's cross-tenant routing test, tenantprovision's INSERT regression test) -- no Docker/database access in this environment; they're correct Go that has never executed, disclosed as such in docs/security/ threat-model.md and docs/phase-4-runbook.md §14.
This commit is contained in:
@@ -0,0 +1,101 @@
|
||||
// Package ingestconfig loads enterprise-ingest's configuration from
|
||||
// environment variables -- same convention as every other Go service in
|
||||
// this repo. Named ingestconfig, not config, to avoid colliding with
|
||||
// enterprise/internal/config (enterprise-auth's own, differently-shaped
|
||||
// config) within the same module -- mirrors enterprise/internal/
|
||||
// apiconfig's own naming reasoning exactly.
|
||||
package ingestconfig
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
// HTTPListenAddr serves only /healthz -- this binary's actual job
|
||||
// (Redpanda -> per-tenant ClickHouse) has no other HTTP surface,
|
||||
// same "just enough for Docker's HEALTHCHECK" shape as every other
|
||||
// binary in this repo's -healthcheck self-check mode.
|
||||
HTTPListenAddr string
|
||||
// ClickHouseAddr is the shared physical ClickHouse server's native
|
||||
// address -- every tenant's connection (enterprise/internal/
|
||||
// chwriter.Registry) dials this same address, just with different
|
||||
// per-tenant credentials rbacstore already has on file from
|
||||
// enterprise-api -provision-tenant. Mirrors apiconfig.Config.
|
||||
// ClickHouseAddr's own doc comment.
|
||||
ClickHouseAddr string
|
||||
Postgres PostgresConfig
|
||||
Redpanda RedpandaConfig
|
||||
Batch BatchConfig
|
||||
}
|
||||
|
||||
type PostgresConfig struct {
|
||||
Addr string
|
||||
Database string
|
||||
Username string
|
||||
Password string
|
||||
}
|
||||
|
||||
type RedpandaConfig struct {
|
||||
Brokers []string
|
||||
Topic string
|
||||
ConsumerGroup string
|
||||
}
|
||||
|
||||
type BatchConfig struct {
|
||||
MaxSize int
|
||||
FlushIntervalMS int
|
||||
}
|
||||
|
||||
func Load() (Config, error) {
|
||||
cfg := Config{
|
||||
HTTPListenAddr: getenv("HTTP_LISTEN_ADDR", ":8084"),
|
||||
ClickHouseAddr: getenv("CLICKHOUSE_ADDR", "localhost:9000"),
|
||||
Postgres: PostgresConfig{
|
||||
Addr: getenv("POSTGRES_ADDR", "localhost:5432"),
|
||||
Database: getenv("POSTGRES_DATABASE", "sentry_metadata"),
|
||||
Username: getenv("POSTGRES_USERNAME", "sentry"),
|
||||
Password: getenv("POSTGRES_PASSWORD", ""),
|
||||
},
|
||||
Redpanda: RedpandaConfig{
|
||||
Brokers: strings.Split(getenv("REDPANDA_BROKERS", "localhost:9092"), ","),
|
||||
// Same default topic ingest/internal/config uses -- this
|
||||
// binary reads the identical shared sentry.logs.raw topic
|
||||
// ingest/cmd/ingest's server half (agent-facing PushBatch)
|
||||
// produces onto; there's no per-tenant topic, see
|
||||
// ingest/internal/grpcserver's doc comment.
|
||||
Topic: getenv("REDPANDA_TOPIC", "sentry.logs.raw"),
|
||||
// A distinct consumer group from ingest/cmd/ingest's own
|
||||
// default ("sentry-ingest") -- this binary and a
|
||||
// single-tenant `ingest -mode=consumer` must never share a
|
||||
// group (each message would only ever reach one of them,
|
||||
// silently splitting traffic) even though in practice a
|
||||
// real multi-tenant deployment runs this binary *instead
|
||||
// of*, not alongside, `ingest -mode=consumer`.
|
||||
ConsumerGroup: getenv("REDPANDA_CONSUMER_GROUP", "sentry-enterprise-ingest"),
|
||||
},
|
||||
}
|
||||
|
||||
maxSize, err := strconv.Atoi(getenv("CONSUMER_BATCH_MAX_SIZE", "500"))
|
||||
if err != nil {
|
||||
return Config{}, fmt.Errorf("CONSUMER_BATCH_MAX_SIZE: %w", err)
|
||||
}
|
||||
cfg.Batch.MaxSize = maxSize
|
||||
|
||||
flushMS, err := strconv.Atoi(getenv("CONSUMER_BATCH_FLUSH_INTERVAL_MS", "2000"))
|
||||
if err != nil {
|
||||
return Config{}, fmt.Errorf("CONSUMER_BATCH_FLUSH_INTERVAL_MS: %w", err)
|
||||
}
|
||||
cfg.Batch.FlushIntervalMS = flushMS
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func getenv(key, fallback string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
Reference in New Issue
Block a user