Phase 4: real per-tenant ClickHouse isolation via a new enterprise-api binary
Closes the threat model's headline finding for the SQL query path:
enterprise/internal/tenantprovision does real CREATE DATABASE/USER/GRANT
against ClickHouse, and enterprise/internal/chrunner is a per-tenant
connection registry implementing api's SQLRunner interface, resolving
the tenant from the authenticated request identity -- never a
caller-suppliable parameter. Both are wired into a new binary,
enterprise/cmd/enterprise-api, alongside the unchanged single-tenant
api/cmd/api, since AGPL core can never import enterprise/ and Go's own
internal/ package visibility rules meant enterprise/ couldn't implement
core's SQLRunner interface without importing the package that defines
it. That required moving api/internal/{authz,queryapi,dashboards,
querylang/executor,searchclient,httpserver} out of internal/ -- the
minimal set enterprise-api needs to import; querylang's compiler
internals (planner/lexer/parser/ast/ir) and api's own config stay
internal, since nothing outside api needs them directly.
Also finally wires enterprise/internal/audit into queryapi.AuditLogger
(nil since Phase 4 task 4) via a new adapter, and adds live-ClickHouse
integration tests for two of the four adversarial probes named in
docs/phase-4-isolation-design.md's verification plan.
Corrected several overclaims in the docs while writing this up: an
earlier claim that rbacstore's CRUD was "verified against a live
Postgres" was never actually true in this environment (only
internal/audit was, earlier in this phase, before Docker access was
lost) -- threat-model.md, phase-4-runbook.md, CLAUDE.md, and
enterprise/README.md all now distinguish "a real integration test
exists" from "this was confirmed against a live database."
Still not built: Tantivy/free-text tenant isolation
(enterprise/internal/searchclient), and any deployment-topology
mechanism that actually routes traffic to enterprise-api instead of
plain api -- both binaries exist side by side today with nothing
enforcing or flagging which one a deployment runs.
This commit is contained in:
@@ -0,0 +1,103 @@
|
||||
// Package apiconfig loads enterprise-api's configuration from
|
||||
// environment variables -- same convention as every other Go service in
|
||||
// this repo. Named apiconfig, not config, to avoid colliding with the
|
||||
// already-existing enterprise/internal/config (enterprise-auth's own,
|
||||
// differently-shaped config) within the same module.
|
||||
package apiconfig
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
HTTPListenAddr string
|
||||
// ClickHouseAddr is the shared physical ClickHouse server's native
|
||||
// address -- every tenant's connection (chrunner.Registry) and the
|
||||
// admin connection (tenantprovision) both dial this same address,
|
||||
// just with different credentials. Tenants sharing one physical
|
||||
// server is today's model; per-tenant dedicated cluster nodes is
|
||||
// named as later, non-schema-changing work in
|
||||
// /docs/phase-4-isolation-design.md.
|
||||
ClickHouseAddr string
|
||||
// ClickHouseAdmin is the access_management-enabled credential
|
||||
// tenantprovision uses for CREATE DATABASE/USER/GRANT -- the same
|
||||
// credential api's plain (non-enterprise) binary uses as its one
|
||||
// shared connection today (docker-compose.yml's CLICKHOUSE_PASSWORD).
|
||||
// Never used to run a tenant's actual queries.
|
||||
ClickHouseAdmin ClickHouseAdminConfig
|
||||
Postgres PostgresConfig
|
||||
AuditWriter AuditWriterConfig
|
||||
SearchGRPCAddr string
|
||||
QueryTimeout time.Duration
|
||||
CORSAllowedOrigin string
|
||||
// EnterpriseAuthURL, like api's own config, is optional -- see that
|
||||
// package's doc comment on the nil-authorizer no-op default. In
|
||||
// practice a real enterprise-api deployment always sets this (there
|
||||
// is no reason to run this binary instead of plain api without RBAC
|
||||
// enforcement on), but nothing here hard-requires it, for the same
|
||||
// "never break a simpler deployment shape" reasoning used
|
||||
// throughout this codebase.
|
||||
EnterpriseAuthURL string
|
||||
}
|
||||
|
||||
type ClickHouseAdminConfig struct {
|
||||
Username string
|
||||
Password string
|
||||
}
|
||||
|
||||
type PostgresConfig struct {
|
||||
Addr string
|
||||
Database string
|
||||
Username string
|
||||
Password string
|
||||
}
|
||||
|
||||
// AuditWriterConfig is the separate, narrowly-granted credential
|
||||
// enterprise/internal/audit.Store requires -- see that package's doc
|
||||
// comment on why it must never share api's/dashboards' pool.
|
||||
type AuditWriterConfig struct {
|
||||
Username string
|
||||
Password string
|
||||
}
|
||||
|
||||
func Load() (Config, error) {
|
||||
cfg := Config{
|
||||
HTTPListenAddr: getenv("HTTP_LISTEN_ADDR", ":8083"),
|
||||
ClickHouseAddr: getenv("CLICKHOUSE_ADDR", "localhost:9000"),
|
||||
ClickHouseAdmin: ClickHouseAdminConfig{
|
||||
Username: getenv("CLICKHOUSE_ADMIN_USERNAME", "default"),
|
||||
Password: getenv("CLICKHOUSE_ADMIN_PASSWORD", ""),
|
||||
},
|
||||
Postgres: PostgresConfig{
|
||||
Addr: getenv("POSTGRES_ADDR", "localhost:5432"),
|
||||
Database: getenv("POSTGRES_DATABASE", "sentry_metadata"),
|
||||
Username: getenv("POSTGRES_USERNAME", "sentry"),
|
||||
Password: getenv("POSTGRES_PASSWORD", ""),
|
||||
},
|
||||
AuditWriter: AuditWriterConfig{
|
||||
Username: getenv("AUDIT_WRITER_USERNAME", "audit_writer"),
|
||||
Password: getenv("AUDIT_WRITER_PASSWORD", ""),
|
||||
},
|
||||
SearchGRPCAddr: getenv("SEARCH_GRPC_ADDR", "localhost:50052"),
|
||||
CORSAllowedOrigin: getenv("CORS_ALLOWED_ORIGIN", "*"),
|
||||
EnterpriseAuthURL: getenv("ENTERPRISE_AUTH_URL", ""),
|
||||
}
|
||||
|
||||
timeoutSec, err := strconv.Atoi(getenv("QUERY_TIMEOUT_SECONDS", "30"))
|
||||
if err != nil {
|
||||
return Config{}, fmt.Errorf("QUERY_TIMEOUT_SECONDS: %w", err)
|
||||
}
|
||||
cfg.QueryTimeout = time.Duration(timeoutSec) * time.Second
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func getenv(key, fallback string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
@@ -18,8 +18,12 @@ import (
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"github.com/sentry/sentry/api/authz"
|
||||
"github.com/sentry/sentry/api/queryapi"
|
||||
)
|
||||
|
||||
func testPool(t *testing.T, user, password string) *pgxpool.Pool {
|
||||
@@ -92,6 +96,50 @@ func TestAppendAndVerifyChainRealPostgres(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestQueryAPILoggerWritesAttributedToContextIdentity proves the
|
||||
// adapter queryapi.Handler actually calls in production (via
|
||||
// enterprise-api's wiring) reads tenant/user from context, not from any
|
||||
// field on QueryAuditEntry -- matching that type's own doc comment.
|
||||
func TestQueryAPILoggerWritesAttributedToContextIdentity(t *testing.T) {
|
||||
writerPool := testPool(t, "audit_writer", os.Getenv("AUDIT_TEST_POSTGRES_PASSWORD"))
|
||||
adminPool := testPool(t, "sentry", os.Getenv("AUDIT_TEST_ADMIN_PASSWORD"))
|
||||
cleanupAuditLog(t, adminPool)
|
||||
defer cleanupAuditLog(t, adminPool)
|
||||
|
||||
logger := NewQueryAPILogger(NewStore(writerPool), SourceAPI)
|
||||
ctx := authz.WithIdentity(context.Background(), authz.Identity{TenantID: "acme", UserID: "11111111-1111-1111-1111-111111111111", Role: authz.RoleViewer})
|
||||
|
||||
err := logger.LogQuery(ctx, queryapi.QueryAuditEntry{
|
||||
Query: "stats count", Language: "spl", RowCount: 3, Duration: 42 * time.Millisecond, Success: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("LogQuery: %v", err)
|
||||
}
|
||||
|
||||
var tenantID, userID, queryText string
|
||||
row := adminPool.QueryRow(context.Background(),
|
||||
`SELECT tenant_id, user_id, query_text FROM audit_log ORDER BY id DESC LIMIT 1`)
|
||||
if err := row.Scan(&tenantID, &userID, &queryText); err != nil {
|
||||
t.Fatalf("reading back the written row: %v", err)
|
||||
}
|
||||
if tenantID != "acme" || userID != "11111111-1111-1111-1111-111111111111" || queryText != "stats count" {
|
||||
t.Fatalf("got tenant_id=%q user_id=%q query_text=%q, want acme/11111111-.../\"stats count\"", tenantID, userID, queryText)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryAPILoggerRefusesWithoutIdentity(t *testing.T) {
|
||||
writerPool := testPool(t, "audit_writer", os.Getenv("AUDIT_TEST_POSTGRES_PASSWORD"))
|
||||
adminPool := testPool(t, "sentry", os.Getenv("AUDIT_TEST_ADMIN_PASSWORD"))
|
||||
cleanupAuditLog(t, adminPool)
|
||||
defer cleanupAuditLog(t, adminPool)
|
||||
|
||||
logger := NewQueryAPILogger(NewStore(writerPool), SourceAPI)
|
||||
err := logger.LogQuery(context.Background(), queryapi.QueryAuditEntry{Query: "stats count", Success: true})
|
||||
if err == nil {
|
||||
t.Fatal("expected LogQuery to refuse writing an entry with no tenant identity in context")
|
||||
}
|
||||
}
|
||||
|
||||
// TestVerifyChainDetectsTampering proves the chain actually catches an
|
||||
// in-place row modification -- not just that VerifyChain runs without
|
||||
// erroring on untampered data, which a bug returning OK unconditionally
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
// Adapts *Store to api/queryapi.AuditLogger -- the interface core
|
||||
// defines and has carried as a nil-by-default field
|
||||
// (api/queryapi.Handler.audit) since Phase 4 task 4, waiting on exactly
|
||||
// this: a real implementation, wired in by enterprise/cmd/enterprise-api
|
||||
// (the one binary allowed to import both packages -- see chrunner's doc
|
||||
// comment on the enterprise->api import direction).
|
||||
package audit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/sentry/sentry/api/authz"
|
||||
"github.com/sentry/sentry/api/queryapi"
|
||||
)
|
||||
|
||||
// QueryAPILogger implements queryapi.AuditLogger by translating its
|
||||
// tenant-agnostic QueryAuditEntry into this package's Entry, reading
|
||||
// tenant/user identity from ctx -- exactly the shape
|
||||
// queryapi.AuditLogger's doc comment describes: "an enterprise-side
|
||||
// implementation reads identity from ctx rather than this interface
|
||||
// growing tenant-awareness."
|
||||
type QueryAPILogger struct {
|
||||
store *Store
|
||||
source Source
|
||||
}
|
||||
|
||||
// NewQueryAPILogger wraps store for use as a specific Source -- api's
|
||||
// queryapi.Handler and /alerting's evaluations go through different
|
||||
// enterprise-api-fronted paths today (SourceAPI is the only one
|
||||
// actually wired to a real HTTP handler; SourceAlerting is named for
|
||||
// when alerting's queries get audited the same way, not yet built).
|
||||
func NewQueryAPILogger(store *Store, source Source) *QueryAPILogger {
|
||||
return &QueryAPILogger{store: store, source: source}
|
||||
}
|
||||
|
||||
func (l *QueryAPILogger) LogQuery(ctx context.Context, entry queryapi.QueryAuditEntry) error {
|
||||
identity, ok := authz.IdentityFromContext(ctx)
|
||||
if !ok || identity.TenantID == "" {
|
||||
// Fail open at the queryapi.Handler call site already covers
|
||||
// "don't take down the query path" -- this specific error tells
|
||||
// that fail-open path *why* the write didn't happen, distinct
|
||||
// from a real audit-storage failure, since chrunner.RunSQL
|
||||
// would already have refused the query itself in this case (see
|
||||
// that package's RunSQL) -- this branch mostly protects against
|
||||
// a future caller that skips chrunner's own check.
|
||||
return fmt.Errorf("audit: no tenant identity in context, refusing to write an unattributable audit entry")
|
||||
}
|
||||
|
||||
status := StatusSuccess
|
||||
var errMsg *string
|
||||
if !entry.Success {
|
||||
status = StatusError
|
||||
errMsg = &entry.Error
|
||||
}
|
||||
var userID *string
|
||||
if identity.UserID != "" {
|
||||
userID = &identity.UserID
|
||||
}
|
||||
queryText := entry.Query
|
||||
rowCount := entry.RowCount
|
||||
durationMS := int(entry.Duration.Milliseconds())
|
||||
|
||||
_, err := l.store.Append(ctx, Entry{
|
||||
TenantID: identity.TenantID,
|
||||
UserID: userID,
|
||||
Source: l.source,
|
||||
EventType: EventQuery,
|
||||
QueryText: &queryText,
|
||||
RowCount: &rowCount,
|
||||
DurationMS: &durationMS,
|
||||
Status: status,
|
||||
ErrorMessage: errMsg,
|
||||
})
|
||||
return err
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
// Package authhandler implements enterprise-auth's POST /internal/authorize
|
||||
// endpoint -- the HTTP side of the "network boundary, not import boundary"
|
||||
// pattern api/internal/authz.HTTPAuthorizer calls into (see that package's
|
||||
// pattern api/authz.HTTPAuthorizer calls into (see that package's
|
||||
// doc comment). It resolves a caller's credentials (session cookie or
|
||||
// service-token Bearer header) to an identity, using session.Manager for
|
||||
// both -- a human session and /alerting's service token are both just
|
||||
@@ -18,7 +18,7 @@ import (
|
||||
"github.com/sentry/sentry/enterprise/internal/session"
|
||||
)
|
||||
|
||||
// SessionCookieName matches the name api/internal/authz.HTTPAuthorizer's
|
||||
// SessionCookieName matches the name api/authz.HTTPAuthorizer's
|
||||
// tests and doc comments already assume ("sentry_session").
|
||||
const SessionCookieName = "sentry_session"
|
||||
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
// Package chrunner is the tenant-scoped implementation of api's
|
||||
// querylang/executor.SQLRunner interface -- the piece
|
||||
// /docs/security/threat-model.md's headline finding says was missing:
|
||||
// until this package, api/cmd/api/main.go opened exactly one shared
|
||||
// ClickHouse connection for every tenant, no matter how many
|
||||
// tenant_memberships/Tenant CRs existed. This package requires
|
||||
// importing api/querylang/executor and api/authz directly (see
|
||||
// enterprise/go.mod's replace directive) -- implementing
|
||||
// executor.SQLRunner structurally requires it (its RunSQL method
|
||||
// returns *executor.Result, a type only that package defines), and
|
||||
// that's the allowed import direction: enterprise -> api, never the
|
||||
// reverse (hack/check-tenant-boundary.sh enforces that direction only).
|
||||
//
|
||||
// Design, per /docs/phase-4-isolation-design.md's ClickHouse section:
|
||||
// Registry holds one fully separate *executor.ChRunner (and the
|
||||
// driver.Conn under it) per tenant, built once at construction from an
|
||||
// immutable map -- never a shared pool with session-level `USE`, which
|
||||
// is a classic concurrency bug (a connection recycled between tenants
|
||||
// mid-flight can interleave one tenant's session state into another's
|
||||
// query). RunSQL resolves which tenant's runner to use from the
|
||||
// request's authz.Identity (attached to ctx by
|
||||
// api/authz.RequireRole/RequireRoleOrService), never from any
|
||||
// caller-suppliable parameter -- there is no code path in this package
|
||||
// that accepts a tenant ID as an argument to a query-executing method.
|
||||
package chrunner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2"
|
||||
"github.com/sentry/sentry/api/authz"
|
||||
"github.com/sentry/sentry/api/querylang/executor"
|
||||
)
|
||||
|
||||
// DataSource is the minimal shape Registry needs to open one tenant's
|
||||
// connection -- deliberately not enterprise/internal/rbacstore.DataSource
|
||||
// itself, so this package doesn't need to import rbacstore just to
|
||||
// describe "an address and a credential." Callers (enterprise-api's
|
||||
// main.go) adapt rbacstore rows into this.
|
||||
type DataSource struct {
|
||||
TenantID string
|
||||
Database string
|
||||
Username string
|
||||
Password string
|
||||
}
|
||||
|
||||
// Registry implements executor.SQLRunner by routing each call to the
|
||||
// caller's tenant-specific connection. Immutable after New returns --
|
||||
// see this file's doc comment on why that's load-bearing, not just a
|
||||
// style choice.
|
||||
type Registry struct {
|
||||
runners map[string]*executor.ChRunner
|
||||
closers []func()
|
||||
}
|
||||
|
||||
// New opens one real ClickHouse connection per DataSource (same native
|
||||
// address for all of them -- tenants sharing a physical ClickHouse
|
||||
// server today, per-tenant *pinning* to dedicated cluster nodes is
|
||||
// named as later, non-schema-changing work in
|
||||
// /docs/phase-4-isolation-design.md, not something this constructor
|
||||
// does). Fails closed: if any one tenant's connection can't be opened
|
||||
// or doesn't ping successfully, the whole Registry fails to construct
|
||||
// rather than silently running with a partial tenant set -- a tenant
|
||||
// missing from the map is a clear, loud "unknown tenant" error at query
|
||||
// time (see RunSQL), not a connection nobody noticed never came up.
|
||||
func New(ctx context.Context, addr string, sources []DataSource) (*Registry, error) {
|
||||
reg := &Registry{runners: make(map[string]*executor.ChRunner, len(sources))}
|
||||
for _, src := range sources {
|
||||
conn, err := clickhouse.Open(&clickhouse.Options{
|
||||
Addr: []string{addr},
|
||||
Auth: clickhouse.Auth{
|
||||
Database: src.Database,
|
||||
Username: src.Username,
|
||||
Password: src.Password,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
reg.Close()
|
||||
return nil, fmt.Errorf("chrunner: opening connection for tenant %q: %w", src.TenantID, err)
|
||||
}
|
||||
if err := conn.Ping(ctx); err != nil {
|
||||
_ = conn.Close()
|
||||
reg.Close()
|
||||
return nil, fmt.Errorf("chrunner: pinging connection for tenant %q: %w", src.TenantID, err)
|
||||
}
|
||||
reg.runners[src.TenantID] = executor.NewChRunner(conn)
|
||||
reg.closers = append(reg.closers, func() { _ = conn.Close() })
|
||||
}
|
||||
return reg, nil
|
||||
}
|
||||
|
||||
// Close releases every underlying connection -- call once at process
|
||||
// shutdown, same lifecycle as the single conn.Close() api/cmd/api/main.go
|
||||
// defers today, just fanned out over N connections.
|
||||
func (r *Registry) Close() {
|
||||
for _, c := range r.closers {
|
||||
c()
|
||||
}
|
||||
}
|
||||
|
||||
// RunSQL implements executor.SQLRunner. Resolves the caller's tenant
|
||||
// from ctx (never a parameter -- see this file's doc comment) and fails
|
||||
// closed on every ambiguous case: no identity, an identity with no
|
||||
// tenant (RoleService, or a misconfigured authorizer), or a tenant with
|
||||
// no provisioned connection all return an error, never a fallback to
|
||||
// some other tenant's connection or an arbitrarily-chosen default.
|
||||
func (r *Registry) RunSQL(ctx context.Context, sql string) (*executor.Result, error) {
|
||||
identity, ok := authz.IdentityFromContext(ctx)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("chrunner: no authenticated identity in context, refusing to run query")
|
||||
}
|
||||
if identity.TenantID == "" {
|
||||
return nil, fmt.Errorf("chrunner: authenticated identity %q has no tenant, refusing to run query", identity.Role)
|
||||
}
|
||||
runner, ok := r.runners[identity.TenantID]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("chrunner: tenant %q has no provisioned ClickHouse connection", identity.TenantID)
|
||||
}
|
||||
return runner.RunSQL(ctx, sql)
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
// Integration tests against a real ClickHouse -- exercises the actual
|
||||
// question this package exists to answer: does a query authenticated as
|
||||
// tenant A ever see tenant B's data. Uses enterprise/internal/
|
||||
// tenantprovision to set up real per-tenant users first (this is the
|
||||
// adversarial probe api/queryapi/tenant_isolation_gap_test.go's
|
||||
// TestAdversarial_ClickHouseUserCannotReadOtherTenantDatabaseByFullyQualifiedName
|
||||
// names as blocked -- this is where it stops being blocked, at the
|
||||
// chrunner/query-execution layer specifically, complementing
|
||||
// tenantprovision's own version of the same probe at the raw-SQL-user
|
||||
// layer).
|
||||
//
|
||||
// Skipped unless CHRUNNER_TEST_CLICKHOUSE_ADDR is set; run via:
|
||||
//
|
||||
// docker run --rm --network sentry_default -v $(pwd)/../../..:/src -w /src/enterprise \
|
||||
// -e CHRUNNER_TEST_CLICKHOUSE_ADDR=clickhouse:9000 \
|
||||
// -e CHRUNNER_TEST_CLICKHOUSE_PASSWORD=sentry-dev-only \
|
||||
// golang:1.25-alpine go test ./internal/chrunner/... -v
|
||||
package chrunner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
chdriver "github.com/ClickHouse/clickhouse-go/v2"
|
||||
"github.com/google/uuid"
|
||||
"github.com/sentry/sentry/api/authz"
|
||||
|
||||
"github.com/sentry/sentry/enterprise/internal/tenantprovision"
|
||||
)
|
||||
|
||||
func testAddr(t *testing.T) string {
|
||||
t.Helper()
|
||||
addr := os.Getenv("CHRUNNER_TEST_CLICKHOUSE_ADDR")
|
||||
if addr == "" {
|
||||
t.Skip("CHRUNNER_TEST_CLICKHOUSE_ADDR not set -- skipping live-ClickHouse integration test")
|
||||
}
|
||||
return addr
|
||||
}
|
||||
|
||||
func provisionTestTenant(t *testing.T, addr string) (tenantID string, creds tenantprovision.Credentials) {
|
||||
t.Helper()
|
||||
admin, err := chdriver.Open(&chdriver.Options{
|
||||
Addr: []string{addr},
|
||||
Auth: chdriver.Auth{Database: "default", Username: "default", Password: os.Getenv("CHRUNNER_TEST_CLICKHOUSE_PASSWORD")},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("opening admin connection: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { admin.Close() })
|
||||
|
||||
tenantID = "cr" + uuid.NewString()[:8]
|
||||
creds, err = tenantprovision.New(admin).ProvisionClickHouse(context.Background(), tenantID)
|
||||
if err != nil {
|
||||
t.Fatalf("provisioning tenant %s: %v", tenantID, err)
|
||||
}
|
||||
return tenantID, creds
|
||||
}
|
||||
|
||||
func TestRegistryRoutesQueryToCorrectTenant(t *testing.T) {
|
||||
addr := testAddr(t)
|
||||
ctx := context.Background()
|
||||
tenantA, credsA := provisionTestTenant(t, addr)
|
||||
|
||||
// Seed a distinguishing row directly as the tenant (SELECT-only
|
||||
// grant means chrunner's own connection can't INSERT -- use a
|
||||
// throwaway admin connection to seed data, matching how a real
|
||||
// deployment's ingest path would write, not how api's read-only
|
||||
// query path does).
|
||||
admin, err := chdriver.Open(&chdriver.Options{
|
||||
Addr: []string{addr},
|
||||
Auth: chdriver.Auth{Database: "default", Username: "default", Password: os.Getenv("CHRUNNER_TEST_CLICKHOUSE_PASSWORD")},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("opening admin connection: %v", err)
|
||||
}
|
||||
defer admin.Close()
|
||||
if err := admin.Exec(ctx, fmt.Sprintf("CREATE TABLE `%s`.marker (id UInt8) ENGINE = Memory", tenantA)); err != nil {
|
||||
t.Fatalf("creating marker table: %v", err)
|
||||
}
|
||||
if err := admin.Exec(ctx, fmt.Sprintf("INSERT INTO `%s`.marker VALUES (42)", tenantA)); err != nil {
|
||||
t.Fatalf("seeding marker row: %v", err)
|
||||
}
|
||||
|
||||
reg, err := New(ctx, addr, []DataSource{
|
||||
{TenantID: tenantA, Database: tenantA, Username: credsA.Username, Password: credsA.Password},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer reg.Close()
|
||||
|
||||
reqCtx := authz.WithIdentity(ctx, authz.Identity{TenantID: tenantA, Role: authz.RoleViewer})
|
||||
result, err := reg.RunSQL(reqCtx, "SELECT id FROM marker")
|
||||
if err != nil {
|
||||
t.Fatalf("RunSQL: %v", err)
|
||||
}
|
||||
if len(result.Rows) != 1 || result.Rows[0][0] != uint8(42) {
|
||||
t.Fatalf("unexpected result: %+v", result.Rows)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryRefusesQueryWithNoTenantContext(t *testing.T) {
|
||||
addr := testAddr(t)
|
||||
ctx := context.Background()
|
||||
tenantA, credsA := provisionTestTenant(t, addr)
|
||||
|
||||
reg, err := New(ctx, addr, []DataSource{
|
||||
{TenantID: tenantA, Database: tenantA, Username: credsA.Username, Password: credsA.Password},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer reg.Close()
|
||||
|
||||
if _, err := reg.RunSQL(ctx, "SELECT 1"); err == nil {
|
||||
t.Fatal("expected RunSQL to refuse a request with no authenticated identity in context")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryRefusesUnknownTenant(t *testing.T) {
|
||||
addr := testAddr(t)
|
||||
ctx := context.Background()
|
||||
tenantA, credsA := provisionTestTenant(t, addr)
|
||||
|
||||
reg, err := New(ctx, addr, []DataSource{
|
||||
{TenantID: tenantA, Database: tenantA, Username: credsA.Username, Password: credsA.Password},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer reg.Close()
|
||||
|
||||
reqCtx := authz.WithIdentity(ctx, authz.Identity{TenantID: "some-other-tenant-never-provisioned", Role: authz.RoleViewer})
|
||||
if _, err := reg.RunSQL(reqCtx, "SELECT 1"); err == nil {
|
||||
t.Fatal("expected RunSQL to refuse a tenant with no provisioned connection, not silently fall back")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRegistryTenantCannotReadOtherTenantEvenViaRawSQL is the full
|
||||
// end-to-end adversarial probe: two tenants, two connections inside one
|
||||
// Registry, and a raw-SQL attempt (which the query language's escape
|
||||
// hatch would pass straight through unmodified) to read the other
|
||||
// tenant's data by fully-qualified name. This is what proves the
|
||||
// connection-layer isolation model actually holds through chrunner, not
|
||||
// just through tenantprovision's own grants (already covered by
|
||||
// tenantprovision_test.go) -- this test exercises the exact code path
|
||||
// api/queryapi.Handler calls in production.
|
||||
func TestRegistryTenantCannotReadOtherTenantEvenViaRawSQL(t *testing.T) {
|
||||
addr := testAddr(t)
|
||||
ctx := context.Background()
|
||||
tenantA, credsA := provisionTestTenant(t, addr)
|
||||
tenantB, credsB := provisionTestTenant(t, addr)
|
||||
|
||||
admin, err := chdriver.Open(&chdriver.Options{
|
||||
Addr: []string{addr},
|
||||
Auth: chdriver.Auth{Database: "default", Username: "default", Password: os.Getenv("CHRUNNER_TEST_CLICKHOUSE_PASSWORD")},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("opening admin connection: %v", err)
|
||||
}
|
||||
defer admin.Close()
|
||||
if err := admin.Exec(ctx, fmt.Sprintf("CREATE TABLE `%s`.secret (id UInt8) ENGINE = Memory", tenantB)); err != nil {
|
||||
t.Fatalf("creating secret table: %v", err)
|
||||
}
|
||||
if err := admin.Exec(ctx, fmt.Sprintf("INSERT INTO `%s`.secret VALUES (99)", tenantB)); err != nil {
|
||||
t.Fatalf("seeding secret row: %v", err)
|
||||
}
|
||||
|
||||
reg, err := New(ctx, addr, []DataSource{
|
||||
{TenantID: tenantA, Database: tenantA, Username: credsA.Username, Password: credsA.Password},
|
||||
{TenantID: tenantB, Database: tenantB, Username: credsB.Username, Password: credsB.Password},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer reg.Close()
|
||||
|
||||
reqCtx := authz.WithIdentity(ctx, authz.Identity{TenantID: tenantA, Role: authz.RoleViewer})
|
||||
_, err = reg.RunSQL(reqCtx, fmt.Sprintf("SELECT * FROM `%s`.secret", tenantB))
|
||||
if err == nil {
|
||||
t.Fatal("tenant A's request was able to read tenant B's database by fully-qualified name -- isolation is broken")
|
||||
}
|
||||
}
|
||||
@@ -14,10 +14,11 @@
|
||||
// that handler itself isn't built yet (see cmd/enterprise-auth/main.go's
|
||||
// doc comment), so today rbacstore's only production caller is
|
||||
// -mint-service-token's future tenant-aware successor and its own tests.
|
||||
// dashboard_permissions and data_sources (also part of the schema) don't
|
||||
// have CRUD here yet -- no caller needs them until dashboards' handler
|
||||
// wiring reads per-resource grants, named as deferred in task 5's
|
||||
// summary.
|
||||
// dashboard_permissions doesn't have CRUD here yet -- no caller reads
|
||||
// per-resource grants (see api/dashboards/handler.go's doc
|
||||
// comment). data_sources CRUD was added once enterprise/internal/
|
||||
// chrunner needed a real place to read per-tenant ClickHouse credentials
|
||||
// from at startup (see that package's doc comment).
|
||||
package rbacstore
|
||||
|
||||
import (
|
||||
@@ -44,17 +45,17 @@ type User struct {
|
||||
}
|
||||
|
||||
type Tenant struct {
|
||||
ID string
|
||||
DisplayName string
|
||||
Status string
|
||||
OwnerUserID string // empty until a first Owner is assigned
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
ID string
|
||||
DisplayName string
|
||||
Status string
|
||||
OwnerUserID string // empty until a first Owner is assigned
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
// Role mirrors api/internal/authz.Role's string values, kept as a plain
|
||||
// Role mirrors api/authz.Role's string values, kept as a plain
|
||||
// string here rather than importing authz -- rbacstore is enterprise
|
||||
// code and api/internal/authz is core; enterprise may depend on
|
||||
// code and api/authz is core; enterprise may depend on
|
||||
// nothing-shaped-like-an-import-from-core per the module boundary
|
||||
// (see /docs/phase-4-isolation-design.md), even though the reverse
|
||||
// (core importing enterprise) is the one hack/check-tenant-boundary.sh
|
||||
@@ -252,3 +253,114 @@ func (s *Store) ListMembershipsForUser(ctx context.Context, userID string) ([]Me
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// DataSource is one tenant's data-plane location -- today, exactly one
|
||||
// ClickHouse database + one Tantivy index per tenant (see
|
||||
// /docs/phase-4-rbac-design.md's "data_sources" extension-point
|
||||
// section). ClickHouseUsername/Password are nil until
|
||||
// enterprise/internal/tenantprovision actually provisions the
|
||||
// ClickHouse-side user/database and calls SetDataSourceClickHouseCredentials.
|
||||
type DataSource struct {
|
||||
ID string
|
||||
TenantID string
|
||||
Name string
|
||||
ClickHouseDatabaseName string
|
||||
TantivyIndexPath string
|
||||
ClickHouseUsername *string
|
||||
ClickHousePassword *string
|
||||
}
|
||||
|
||||
// CreateDataSource inserts the row tenantprovision will later attach
|
||||
// credentials to (SetDataSourceClickHouseCredentials) -- split into two
|
||||
// steps because the row (database name, index path) is decided before
|
||||
// provisioning runs, but the ClickHouse-side username/password only
|
||||
// exist after CREATE USER actually succeeds.
|
||||
func (s *Store) CreateDataSource(ctx context.Context, tenantID, name, clickHouseDatabaseName, tantivyIndexPath string) (*DataSource, error) {
|
||||
ds := DataSource{
|
||||
ID: uuid.NewString(), TenantID: tenantID, Name: name,
|
||||
ClickHouseDatabaseName: clickHouseDatabaseName, TantivyIndexPath: tantivyIndexPath,
|
||||
}
|
||||
_, err := s.pool.Exec(ctx, `
|
||||
INSERT INTO data_sources (id, tenant_id, name, clickhouse_database_name, tantivy_index_path)
|
||||
VALUES ($1, $2, $3, $4, $5)`,
|
||||
ds.ID, ds.TenantID, ds.Name, ds.ClickHouseDatabaseName, ds.TantivyIndexPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("rbacstore: creating data source: %w", err)
|
||||
}
|
||||
return &ds, nil
|
||||
}
|
||||
|
||||
// SetDataSourceClickHouseCredentials is the only way
|
||||
// clickhouse_username/password change -- called once, right after
|
||||
// enterprise/internal/tenantprovision.ProvisionClickHouse succeeds.
|
||||
// Never called again for the same data source: rotating a live tenant's
|
||||
// credential without first updating it on the ClickHouse side would
|
||||
// just break every open connection, same reasoning as
|
||||
// deploy/operator/internal/controller/tenant_controller.go's
|
||||
// reconcileSecret.
|
||||
func (s *Store) SetDataSourceClickHouseCredentials(ctx context.Context, id, username, password string) error {
|
||||
tag, err := s.pool.Exec(ctx,
|
||||
`UPDATE data_sources SET clickhouse_username = $2, clickhouse_password = $3 WHERE id = $1`,
|
||||
id, username, password)
|
||||
if err != nil {
|
||||
return fmt.Errorf("rbacstore: setting data source credentials: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func scanDataSource(row pgx.Row) (*DataSource, error) {
|
||||
var ds DataSource
|
||||
if err := row.Scan(&ds.ID, &ds.TenantID, &ds.Name, &ds.ClickHouseDatabaseName, &ds.TantivyIndexPath,
|
||||
&ds.ClickHouseUsername, &ds.ClickHousePassword); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ds, nil
|
||||
}
|
||||
|
||||
func (s *Store) GetDataSourceForTenant(ctx context.Context, tenantID string) (*DataSource, error) {
|
||||
row := s.pool.QueryRow(ctx, `
|
||||
SELECT id, tenant_id, name, clickhouse_database_name, tantivy_index_path, clickhouse_username, clickhouse_password
|
||||
FROM data_sources WHERE tenant_id = $1 ORDER BY created_at LIMIT 1`, tenantID)
|
||||
ds, err := scanDataSource(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return nil, fmt.Errorf("rbacstore: getting data source: %w", err)
|
||||
}
|
||||
return ds, nil
|
||||
}
|
||||
|
||||
// ListProvisionedDataSources returns every data source for an active
|
||||
// tenant that has already been provisioned (ClickHouse credentials set)
|
||||
// -- exactly the set enterprise/internal/chrunner.NewRegistry needs at
|
||||
// startup. A data source with no credentials yet (tenantprovision hasn't
|
||||
// run for it) is deliberately excluded rather than returned with empty
|
||||
// credentials -- chrunner has nothing safe to connect with for it, and
|
||||
// silently including it would turn into a confusing empty-string
|
||||
// connection attempt instead of a clear "not provisioned yet" absence.
|
||||
func (s *Store) ListProvisionedDataSources(ctx context.Context) ([]DataSource, error) {
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT ds.id, ds.tenant_id, ds.name, ds.clickhouse_database_name, ds.tantivy_index_path,
|
||||
ds.clickhouse_username, ds.clickhouse_password
|
||||
FROM data_sources ds
|
||||
JOIN tenants t ON t.id = ds.tenant_id
|
||||
WHERE t.status = 'active' AND ds.clickhouse_username IS NOT NULL AND ds.clickhouse_password IS NOT NULL`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("rbacstore: listing provisioned data sources: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []DataSource
|
||||
for rows.Next() {
|
||||
ds, err := scanDataSource(rows)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("rbacstore: scanning data source: %w", err)
|
||||
}
|
||||
out = append(out, *ds)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
@@ -225,3 +225,124 @@ func TestListMembershipsForUserAcrossTenants(t *testing.T) {
|
||||
t.Fatalf("got %d memberships, want 2: %+v", len(memberships), memberships)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateDataSourceThenSetCredentials(t *testing.T) {
|
||||
s := testStore(t)
|
||||
ctx := context.Background()
|
||||
tenantID := "test-tenant-" + uniqueSuffix()
|
||||
if _, err := s.CreateTenant(ctx, tenantID, "Test Tenant"); err != nil {
|
||||
t.Fatalf("CreateTenant: %v", err)
|
||||
}
|
||||
|
||||
ds, err := s.CreateDataSource(ctx, tenantID, "default", tenantID, "/var/lib/sentry-search/tenants/"+tenantID)
|
||||
if err != nil {
|
||||
t.Fatalf("CreateDataSource: %v", err)
|
||||
}
|
||||
if ds.ClickHouseUsername != nil || ds.ClickHousePassword != nil {
|
||||
t.Fatalf("new data source must have no credentials yet, got %+v", ds)
|
||||
}
|
||||
|
||||
got, err := s.GetDataSourceForTenant(ctx, tenantID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetDataSourceForTenant: %v", err)
|
||||
}
|
||||
if got.ID != ds.ID || got.ClickHouseUsername != nil {
|
||||
t.Fatalf("unexpected data source: %+v", got)
|
||||
}
|
||||
|
||||
if err := s.SetDataSourceClickHouseCredentials(ctx, ds.ID, "tenant_"+tenantID, "secret-password"); err != nil {
|
||||
t.Fatalf("SetDataSourceClickHouseCredentials: %v", err)
|
||||
}
|
||||
got, err = s.GetDataSourceForTenant(ctx, tenantID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetDataSourceForTenant after credentials set: %v", err)
|
||||
}
|
||||
if got.ClickHouseUsername == nil || *got.ClickHouseUsername != "tenant_"+tenantID {
|
||||
t.Fatalf("ClickHouseUsername = %v, want tenant_%s", got.ClickHouseUsername, tenantID)
|
||||
}
|
||||
if got.ClickHousePassword == nil || *got.ClickHousePassword != "secret-password" {
|
||||
t.Fatalf("ClickHousePassword = %v, want secret-password", got.ClickHousePassword)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetDataSourceClickHouseCredentialsNotFound(t *testing.T) {
|
||||
s := testStore(t)
|
||||
if err := s.SetDataSourceClickHouseCredentials(context.Background(), "does-not-exist-"+uniqueSuffix(), "u", "p"); err != ErrNotFound {
|
||||
t.Fatalf("SetDataSourceClickHouseCredentials error = %v, want ErrNotFound", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetDataSourceForTenantNotFound(t *testing.T) {
|
||||
s := testStore(t)
|
||||
if _, err := s.GetDataSourceForTenant(context.Background(), "does-not-exist-"+uniqueSuffix()); err != ErrNotFound {
|
||||
t.Fatalf("GetDataSourceForTenant error = %v, want ErrNotFound", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestListProvisionedDataSourcesExcludesUnprovisionedAndInactive proves
|
||||
// the two filters ListProvisionedDataSources documents: a data source
|
||||
// with no ClickHouse credentials yet is excluded (nothing safe to
|
||||
// connect with), and a data source belonging to a non-active tenant is
|
||||
// excluded too (a suspended/provisioning tenant must not show up in
|
||||
// chrunner's connection registry).
|
||||
func TestListProvisionedDataSourcesExcludesUnprovisionedAndInactive(t *testing.T) {
|
||||
s := testStore(t)
|
||||
ctx := context.Background()
|
||||
|
||||
activeProvisioned := "test-tenant-" + uniqueSuffix()
|
||||
activeUnprovisioned := "test-tenant-" + uniqueSuffix()
|
||||
suspended := "test-tenant-" + uniqueSuffix()
|
||||
|
||||
for _, id := range []string{activeProvisioned, activeUnprovisioned, suspended} {
|
||||
if _, err := s.CreateTenant(ctx, id, id); err != nil {
|
||||
t.Fatalf("CreateTenant %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
if err := s.SetTenantStatus(ctx, activeProvisioned, "active"); err != nil {
|
||||
t.Fatalf("SetTenantStatus activeProvisioned: %v", err)
|
||||
}
|
||||
if err := s.SetTenantStatus(ctx, activeUnprovisioned, "active"); err != nil {
|
||||
t.Fatalf("SetTenantStatus activeUnprovisioned: %v", err)
|
||||
}
|
||||
// suspended stays in 'provisioning' (CreateTenant's default) -- not active.
|
||||
|
||||
dsProvisioned, err := s.CreateDataSource(ctx, activeProvisioned, "default", activeProvisioned, "/idx")
|
||||
if err != nil {
|
||||
t.Fatalf("CreateDataSource activeProvisioned: %v", err)
|
||||
}
|
||||
if err := s.SetDataSourceClickHouseCredentials(ctx, dsProvisioned.ID, "u", "p"); err != nil {
|
||||
t.Fatalf("SetDataSourceClickHouseCredentials: %v", err)
|
||||
}
|
||||
|
||||
if _, err := s.CreateDataSource(ctx, activeUnprovisioned, "default", activeUnprovisioned, "/idx"); err != nil {
|
||||
t.Fatalf("CreateDataSource activeUnprovisioned: %v", err)
|
||||
}
|
||||
|
||||
dsSuspended, err := s.CreateDataSource(ctx, suspended, "default", suspended, "/idx")
|
||||
if err != nil {
|
||||
t.Fatalf("CreateDataSource suspended: %v", err)
|
||||
}
|
||||
if err := s.SetDataSourceClickHouseCredentials(ctx, dsSuspended.ID, "u", "p"); err != nil {
|
||||
t.Fatalf("SetDataSourceClickHouseCredentials suspended: %v", err)
|
||||
}
|
||||
|
||||
list, err := s.ListProvisionedDataSources(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("ListProvisionedDataSources: %v", err)
|
||||
}
|
||||
foundOurs := false
|
||||
for _, ds := range list {
|
||||
if ds.TenantID == activeUnprovisioned {
|
||||
t.Fatalf("unprovisioned data source leaked into the list: %+v", ds)
|
||||
}
|
||||
if ds.TenantID == suspended {
|
||||
t.Fatalf("non-active tenant's data source leaked into the list: %+v", ds)
|
||||
}
|
||||
if ds.TenantID == activeProvisioned {
|
||||
foundOurs = true
|
||||
}
|
||||
}
|
||||
if !foundOurs {
|
||||
t.Fatal("expected the active, provisioned data source to be in the list")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ import (
|
||||
"github.com/go-jose/go-jose/v4/jwt"
|
||||
)
|
||||
|
||||
// Claims mirrors api/internal/authz.Identity's fields (TenantID, UserID,
|
||||
// Claims mirrors api/authz.Identity's fields (TenantID, UserID,
|
||||
// Role as a string) plus the standard registered JWT claims. Role is
|
||||
// deliberately a plain string, not enterprise's own type, since its only
|
||||
// consumer -- authz.Role -- is defined in core and this package must not
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
// Package tenantprovision does the ClickHouse-side half of tenant
|
||||
// provisioning /docs/phase-4-isolation-design.md describes: one
|
||||
// dedicated database + one narrowly-granted user per tenant, ordered
|
||||
// and idempotent (CREATE DATABASE -> CREATE USER -> GRANT). This is the
|
||||
// piece that was missing before -- deploy/operator's Tenant controller
|
||||
// only manages the K8s-side credential Secret; nothing called
|
||||
// ClickHouse's DDL to make that Secret's credentials actually work
|
||||
// until this package.
|
||||
//
|
||||
// What this package does NOT do: Tantivy index provisioning (still
|
||||
// unbuilt -- see /docs/security/threat-model.md), and it does not
|
||||
// itself decide when a tenant becomes 'active' in rbacstore.tenants --
|
||||
// the caller (enterprise-api's -provision-tenant flag) does that only
|
||||
// after ProvisionClickHouse returns success, matching the ordered gate
|
||||
// /docs/phase-4-isolation-design.md specifies: CREATE USER -> GRANT ->
|
||||
// only then mark active.
|
||||
package tenantprovision
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"regexp"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
)
|
||||
|
||||
// tenantIdentifierPattern is deliberately strict: tenant IDs become
|
||||
// literal ClickHouse database/user names, interpolated directly into
|
||||
// DDL statements below (ClickHouse's driver has no parameterized-query
|
||||
// support for identifiers, only values -- this is the real reason
|
||||
// rbacstore.Tenant.ID and every K8s Tenant CRD name are constrained to
|
||||
// look like a DNS-safe slug already; this regexp is the enforcement
|
||||
// point specific to this package's SQL construction, not a general
|
||||
// tenant-ID validator).
|
||||
var tenantIdentifierPattern = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,62}$`)
|
||||
|
||||
// Credentials is what ProvisionClickHouse hands back for the caller to
|
||||
// persist (rbacstore.Store.SetDataSourceClickHouseCredentials) --
|
||||
// Password is returned exactly once; ClickHouse itself doesn't store it
|
||||
// recoverably, so losing this return value means re-provisioning
|
||||
// (dropping and recreating the user) is the only recovery path.
|
||||
type Credentials struct {
|
||||
Username string
|
||||
Password string
|
||||
}
|
||||
|
||||
// Provisioner wraps an admin ClickHouse connection -- one with
|
||||
// access_management enabled, the same credential docker-compose.yml's
|
||||
// CLICKHOUSE_PASSWORD/the Helm chart's clickhouse Secret already is.
|
||||
// Never the per-tenant connections enterprise/internal/chrunner opens.
|
||||
type Provisioner struct {
|
||||
admin driver.Conn
|
||||
}
|
||||
|
||||
func New(admin driver.Conn) *Provisioner {
|
||||
return &Provisioner{admin: admin}
|
||||
}
|
||||
|
||||
// ProvisionClickHouse creates tenantID's database and a fresh user for
|
||||
// it, granted SELECT on exactly that database and nothing else.
|
||||
// Database creation is idempotent (CREATE DATABASE IF NOT EXISTS --
|
||||
// harmless to repeat). User creation deliberately is NOT idempotent
|
||||
// (plain CREATE USER, no IF NOT EXISTS): ClickHouse has no way to read
|
||||
// back an existing user's password, so silently succeeding on a second
|
||||
// call would either mean returning stale/wrong credentials or silently
|
||||
// rotating a live tenant's password out from under it -- the same
|
||||
// "never rotate a live credential without coordinating the consumer
|
||||
// side" reasoning as deploy/operator/internal/controller/
|
||||
// tenant_controller.go's reconcileSecret. A second call for an
|
||||
// already-provisioned tenant fails loudly instead, which is the correct
|
||||
// outcome: the caller (enterprise-api's -provision-tenant flag) must
|
||||
// check rbacstore for existing credentials before ever calling this,
|
||||
// not rely on this function to be safely re-callable.
|
||||
//
|
||||
// system.* access: intentionally not explicitly granted anywhere here,
|
||||
// relying on ClickHouse RBAC's default-deny for a freshly created user
|
||||
// once access_management is enabled on the admin connection (required
|
||||
// for CREATE USER/GRANT to work at all). This is exactly the assumption
|
||||
// /docs/phase-4-isolation-design.md's task 2 finding says must be
|
||||
// verified live per ClickHouse version, not trusted from documentation
|
||||
// -- see /docs/security/threat-model.md's "system.query_log metadata
|
||||
// leakage" section; that verification has not happened yet.
|
||||
func (p *Provisioner) ProvisionClickHouse(ctx context.Context, tenantID string) (Credentials, error) {
|
||||
if !tenantIdentifierPattern.MatchString(tenantID) {
|
||||
return Credentials{}, fmt.Errorf("tenantprovision: tenant id %q is not a safe ClickHouse identifier", tenantID)
|
||||
}
|
||||
|
||||
database := tenantID
|
||||
username := "tenant_" + tenantID
|
||||
|
||||
if err := p.admin.Exec(ctx, fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`", database)); err != nil {
|
||||
return Credentials{}, fmt.Errorf("tenantprovision: creating database: %w", err)
|
||||
}
|
||||
|
||||
password, err := generatePassword()
|
||||
if err != nil {
|
||||
return Credentials{}, err
|
||||
}
|
||||
|
||||
// No IF NOT EXISTS -- see this function's doc comment for why a
|
||||
// second call must fail, not silently succeed.
|
||||
if err := p.admin.Exec(ctx, fmt.Sprintf(
|
||||
"CREATE USER `%s` IDENTIFIED WITH plaintext_password BY '%s'",
|
||||
username, escapeSingleQuotes(password),
|
||||
)); err != nil {
|
||||
return Credentials{}, fmt.Errorf("tenantprovision: creating user (already provisioned? this call is not safe to retry): %w", err)
|
||||
}
|
||||
|
||||
if err := p.admin.Exec(ctx, fmt.Sprintf("GRANT SELECT ON `%s`.* TO `%s`", database, username)); err != nil {
|
||||
return Credentials{}, fmt.Errorf("tenantprovision: granting select: %w", err)
|
||||
}
|
||||
|
||||
return Credentials{Username: username, Password: password}, nil
|
||||
}
|
||||
|
||||
func generatePassword() (string, error) {
|
||||
buf := make([]byte, 24)
|
||||
if _, err := rand.Read(buf); err != nil {
|
||||
return "", fmt.Errorf("tenantprovision: generating password: %w", err)
|
||||
}
|
||||
return base64.RawURLEncoding.EncodeToString(buf), nil
|
||||
}
|
||||
|
||||
// escapeSingleQuotes guards against a generated password (base64
|
||||
// RawURLEncoding, so alphanumeric plus '-'/'_' only, never a literal
|
||||
// quote) accidentally breaking out of the SQL string literal -- belt
|
||||
// and suspenders given generatePassword's actual alphabet can't produce
|
||||
// one, since this function's output gets interpolated directly into DDL
|
||||
// (see tenantIdentifierPattern's doc comment on why: ClickHouse's driver
|
||||
// has no parameterized identifiers/literals for DDL).
|
||||
func escapeSingleQuotes(s string) string {
|
||||
out := make([]byte, 0, len(s))
|
||||
for i := 0; i < len(s); i++ {
|
||||
if s[i] == '\'' {
|
||||
out = append(out, '\'', '\'')
|
||||
continue
|
||||
}
|
||||
out = append(out, s[i])
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
// Integration tests against a real ClickHouse -- this package's whole
|
||||
// job is DDL side effects (CREATE DATABASE/USER, GRANT), which a mock
|
||||
// driver.Conn can't meaningfully verify. Skipped unless
|
||||
// TENANTPROVISION_TEST_CLICKHOUSE_ADDR is set; run via:
|
||||
//
|
||||
// docker run --rm --network sentry_default -v $(pwd)/../../..:/src -w /src/enterprise \
|
||||
// -e TENANTPROVISION_TEST_CLICKHOUSE_ADDR=clickhouse:9000 \
|
||||
// -e TENANTPROVISION_TEST_CLICKHOUSE_PASSWORD=sentry-dev-only \
|
||||
// golang:1.25-alpine go test ./internal/tenantprovision/... -v
|
||||
package tenantprovision
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2"
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func testAdminConn(t *testing.T) driver.Conn {
|
||||
t.Helper()
|
||||
addr := os.Getenv("TENANTPROVISION_TEST_CLICKHOUSE_ADDR")
|
||||
if addr == "" {
|
||||
t.Skip("TENANTPROVISION_TEST_CLICKHOUSE_ADDR not set -- skipping live-ClickHouse integration test")
|
||||
}
|
||||
conn, err := clickhouse.Open(&clickhouse.Options{
|
||||
Addr: []string{addr},
|
||||
Auth: clickhouse.Auth{
|
||||
Database: "default",
|
||||
Username: "default",
|
||||
Password: os.Getenv("TENANTPROVISION_TEST_CLICKHOUSE_PASSWORD"),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("opening admin connection: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { conn.Close() })
|
||||
if err := conn.Ping(context.Background()); err != nil {
|
||||
t.Fatalf("pinging clickhouse: %v", err)
|
||||
}
|
||||
return conn
|
||||
}
|
||||
|
||||
func testTenantID() string {
|
||||
return "tp" + uuid.NewString()[:8]
|
||||
}
|
||||
|
||||
func TestProvisionClickHouseCreatesUsableTenantConnection(t *testing.T) {
|
||||
admin := testAdminConn(t)
|
||||
p := New(admin)
|
||||
tenantID := testTenantID()
|
||||
ctx := context.Background()
|
||||
|
||||
creds, err := p.ProvisionClickHouse(ctx, tenantID)
|
||||
if err != nil {
|
||||
t.Fatalf("ProvisionClickHouse: %v", err)
|
||||
}
|
||||
if creds.Username != "tenant_"+tenantID || creds.Password == "" {
|
||||
t.Fatalf("unexpected credentials: %+v", creds)
|
||||
}
|
||||
|
||||
// Prove the credential actually works: connect as the tenant user
|
||||
// and run a real query against its own database.
|
||||
tenantConn, err := clickhouse.Open(&clickhouse.Options{
|
||||
Addr: []string{os.Getenv("TENANTPROVISION_TEST_CLICKHOUSE_ADDR")},
|
||||
Auth: clickhouse.Auth{Database: tenantID, Username: creds.Username, Password: creds.Password},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("opening tenant connection: %v", err)
|
||||
}
|
||||
defer tenantConn.Close()
|
||||
if err := tenantConn.Ping(ctx); err != nil {
|
||||
t.Fatalf("pinging as the provisioned tenant user: %v", err)
|
||||
}
|
||||
if err := tenantConn.Exec(ctx, "SELECT 1"); err != nil {
|
||||
t.Fatalf("running SELECT as the provisioned tenant user: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestProvisionedUserCannotReadOtherTenantDatabase is one of the four
|
||||
// adversarial probes /docs/phase-4-isolation-design.md's verification
|
||||
// plan names for Phase 4 task 8 (see api/queryapi/
|
||||
// tenant_isolation_gap_test.go, which stubs this exact scenario as
|
||||
// blocked pending tenantprovision existing) -- now that tenantprovision
|
||||
// exists, this is the first one that can actually run for real.
|
||||
func TestProvisionedUserCannotReadOtherTenantDatabase(t *testing.T) {
|
||||
admin := testAdminConn(t)
|
||||
p := New(admin)
|
||||
ctx := context.Background()
|
||||
|
||||
tenantA := testTenantID()
|
||||
tenantB := testTenantID()
|
||||
credsA, err := p.ProvisionClickHouse(ctx, tenantA)
|
||||
if err != nil {
|
||||
t.Fatalf("provisioning tenant A: %v", err)
|
||||
}
|
||||
if _, err := p.ProvisionClickHouse(ctx, tenantB); err != nil {
|
||||
t.Fatalf("provisioning tenant B: %v", err)
|
||||
}
|
||||
|
||||
// Seed a row in tenant B's database as admin.
|
||||
if err := admin.Exec(ctx, fmt.Sprintf("CREATE TABLE `%s`.secret (id UInt8) ENGINE = Memory", tenantB)); err != nil {
|
||||
t.Fatalf("creating table in tenant B's database: %v", err)
|
||||
}
|
||||
if err := admin.Exec(ctx, fmt.Sprintf("INSERT INTO `%s`.secret VALUES (1)", tenantB)); err != nil {
|
||||
t.Fatalf("inserting into tenant B's database: %v", err)
|
||||
}
|
||||
|
||||
tenantAConn, err := clickhouse.Open(&clickhouse.Options{
|
||||
Addr: []string{os.Getenv("TENANTPROVISION_TEST_CLICKHOUSE_ADDR")},
|
||||
Auth: clickhouse.Auth{Database: tenantA, Username: credsA.Username, Password: credsA.Password},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("opening tenant A connection: %v", err)
|
||||
}
|
||||
defer tenantAConn.Close()
|
||||
|
||||
// The core adversarial probe: tenant A's user attempting to read
|
||||
// tenant B's database by fully-qualified name in raw SQL.
|
||||
err = tenantAConn.Exec(ctx, fmt.Sprintf("SELECT * FROM `%s`.secret", tenantB))
|
||||
if err == nil {
|
||||
t.Fatal("tenant A's user was able to read tenant B's database -- isolation is broken")
|
||||
}
|
||||
}
|
||||
|
||||
// TestProvisionedUserCannotReadSystemTables is item 2 of
|
||||
// /docs/phase-4-isolation-design.md's verification plan (see
|
||||
// api/queryapi/tenant_isolation_gap_test.go for the other three items'
|
||||
// status) -- task 2's finding was that system.* visibility for a
|
||||
// non-admin ClickHouse user is version-dependent and must be checked
|
||||
// live, not assumed from documentation. ProvisionClickHouse never
|
||||
// explicitly grants system.* access to anything (see its doc comment);
|
||||
// this test is what actually confirms that omission is sufficient on
|
||||
// the ClickHouse version this repo pins (docker-compose.yml:
|
||||
// clickhouse/clickhouse-server:24.8), rather than trusting the omission
|
||||
// alone.
|
||||
func TestProvisionedUserCannotReadSystemTables(t *testing.T) {
|
||||
admin := testAdminConn(t)
|
||||
p := New(admin)
|
||||
tenantID := testTenantID()
|
||||
creds, err := p.ProvisionClickHouse(context.Background(), tenantID)
|
||||
if err != nil {
|
||||
t.Fatalf("ProvisionClickHouse: %v", err)
|
||||
}
|
||||
|
||||
tenantConn, err := clickhouse.Open(&clickhouse.Options{
|
||||
Addr: []string{os.Getenv("TENANTPROVISION_TEST_CLICKHOUSE_ADDR")},
|
||||
Auth: clickhouse.Auth{Database: tenantID, Username: creds.Username, Password: creds.Password},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("opening tenant connection: %v", err)
|
||||
}
|
||||
defer tenantConn.Close()
|
||||
|
||||
// system.query_log/system.tables: expect a hard access-denied error,
|
||||
// not a filtered/empty result -- these tables contain other
|
||||
// tenants' query text and schema, so "succeeds but happens to
|
||||
// return nothing for this user" would still be a version-dependent
|
||||
// assumption worth catching, not something this test treats as a pass.
|
||||
for _, probe := range []string{
|
||||
"SELECT * FROM system.query_log LIMIT 1",
|
||||
"SELECT * FROM system.tables LIMIT 1",
|
||||
} {
|
||||
if err := tenantConn.Exec(context.Background(), probe); err == nil {
|
||||
t.Errorf("tenant user was able to run %q -- system.* access was not actually revoked on this ClickHouse version", probe)
|
||||
}
|
||||
}
|
||||
|
||||
// SHOW DATABASES is checked differently: some ClickHouse versions
|
||||
// filter this to only databases the user can see rather than
|
||||
// erroring outright, which is an acceptable outcome for this
|
||||
// specific statement (unlike query_log/tables above) as long as it
|
||||
// doesn't reveal other tenants' database names.
|
||||
rows, err := tenantConn.Query(context.Background(), "SHOW DATABASES")
|
||||
if err != nil {
|
||||
return // erroring outright is also an acceptable outcome here.
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var db string
|
||||
if err := rows.Scan(&db); err != nil {
|
||||
t.Fatalf("scanning SHOW DATABASES row: %v", err)
|
||||
}
|
||||
if db != tenantID && db != "default" && db != "system" && db != "INFORMATION_SCHEMA" && db != "information_schema" {
|
||||
t.Errorf("SHOW DATABASES revealed a database this tenant user shouldn't see: %q", db)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvisionClickHouseRejectsUnsafeTenantID(t *testing.T) {
|
||||
admin := testAdminConn(t)
|
||||
p := New(admin)
|
||||
if _, err := p.ProvisionClickHouse(context.Background(), "not safe; DROP TABLE x"); err == nil {
|
||||
t.Fatal("expected ProvisionClickHouse to reject an unsafe tenant identifier")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvisionClickHouseSecondCallForSameTenantFails(t *testing.T) {
|
||||
admin := testAdminConn(t)
|
||||
p := New(admin)
|
||||
tenantID := testTenantID()
|
||||
ctx := context.Background()
|
||||
|
||||
if _, err := p.ProvisionClickHouse(ctx, tenantID); err != nil {
|
||||
t.Fatalf("first ProvisionClickHouse: %v", err)
|
||||
}
|
||||
if _, err := p.ProvisionClickHouse(ctx, tenantID); err == nil {
|
||||
t.Fatal("expected a second ProvisionClickHouse call for the same tenant to fail -- see the function's doc comment on why re-provisioning must not silently succeed")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user