Phase 4: real per-tenant ClickHouse isolation via a new enterprise-api binary

Closes the threat model's headline finding for the SQL query path:
enterprise/internal/tenantprovision does real CREATE DATABASE/USER/GRANT
against ClickHouse, and enterprise/internal/chrunner is a per-tenant
connection registry implementing api's SQLRunner interface, resolving
the tenant from the authenticated request identity -- never a
caller-suppliable parameter. Both are wired into a new binary,
enterprise/cmd/enterprise-api, alongside the unchanged single-tenant
api/cmd/api, since AGPL core can never import enterprise/ and Go's own
internal/ package visibility rules meant enterprise/ couldn't implement
core's SQLRunner interface without importing the package that defines
it. That required moving api/internal/{authz,queryapi,dashboards,
querylang/executor,searchclient,httpserver} out of internal/ -- the
minimal set enterprise-api needs to import; querylang's compiler
internals (planner/lexer/parser/ast/ir) and api's own config stay
internal, since nothing outside api needs them directly.

Also finally wires enterprise/internal/audit into queryapi.AuditLogger
(nil since Phase 4 task 4) via a new adapter, and adds live-ClickHouse
integration tests for two of the four adversarial probes named in
docs/phase-4-isolation-design.md's verification plan.

Corrected several overclaims in the docs while writing this up: an
earlier claim that rbacstore's CRUD was "verified against a live
Postgres" was never actually true in this environment (only
internal/audit was, earlier in this phase, before Docker access was
lost) -- threat-model.md, phase-4-runbook.md, CLAUDE.md, and
enterprise/README.md all now distinguish "a real integration test
exists" from "this was confirmed against a live database."

Still not built: Tantivy/free-text tenant isolation
(enterprise/internal/searchclient), and any deployment-topology
mechanism that actually routes traffic to enterprise-api instead of
plain api -- both binaries exist side by side today with nothing
enforcing or flagging which one a deployment runs.

api/authz/authz.go

@@ -0,0 +1,87 @@
+// Package authz is core's RBAC extension point -- deliberately minimal
+// and tenant-agnostic in shape, matching queryapi.AuditLogger's pattern:
+// core defines the interface and the types, enterprise/ supplies the
+// implementation. Unlike AuditLogger, the implementation Authorizer
+// wires in here is NOT enterprise Go code injected directly (that would
+// require api to import enterprise/, violating the module boundary
+// confirmed in /docs/phase-4-isolation-design.md) -- it's an HTTP client
+// to enterprise-auth's /internal/authorize endpoint (see httpauthz.go),
+// the same "network boundary, not import boundary" pattern already
+// established between /api and /alerting.
+package authz
+
+import (
+	"context"
+	"net/http"
+)
+
+// Role is ordered for human roles (Viewer < Editor < Admin < Owner, per
+// /docs/phase-4-rbac-design.md) plus a separate, non-comparable Service
+// lane for machine callers like /alerting's evaluator -- see Satisfies.
+type Role string
+
+const (
+	RoleViewer  Role = "viewer"
+	RoleEditor  Role = "editor"
+	RoleAdmin   Role = "admin"
+	RoleOwner   Role = "owner"
+	RoleService Role = "service"
+)
+
+var roleRank = map[Role]int{RoleViewer: 1, RoleEditor: 2, RoleAdmin: 3, RoleOwner: 4}
+
+// Satisfies reports whether this role meets a requirement. RoleService
+// only ever satisfies RoleService -- a service credential never
+// satisfies a human-role requirement, and a human role never satisfies
+// a RoleService requirement, by design: /docs/phase-4-isolation-design.md's
+// alerting service identity is deliberately not a point on the human
+// role scale, so it can't accidentally inherit broader access by
+// ranking above Viewer.
+func (r Role) Satisfies(required Role) bool {
+	if required == RoleService || r == RoleService {
+		return r == required
+	}
+	return roleRank[r] >= roleRank[required]
+}
+
+// Identity is what a successful Authorize call resolves. UserID is
+// empty for RoleService (see /docs/phase-4-isolation-design.md's
+// alerting↔api gap -- a service credential proves "this caller is
+// alerting," not "this caller is acting as a specific human").
+type Identity struct {
+	TenantID string
+	UserID   string
+	Role     Role
+}
+
+// Authorizer resolves an Identity from an incoming request's
+// credentials (session cookie or service token) without knowing
+// anything about *what* the caller is trying to do -- permission
+// checking against a required Role happens in the middleware
+// (middleware.go), not here, so this interface stays a pure
+// "who is this" question.
+type Authorizer interface {
+	Authorize(r *http.Request) (Identity, error)
+}
+
+type identityContextKey struct{}
+
+// IdentityFromContext lets a handler read the resolved identity a
+// RequireRole/RequireRoleOrService middleware attached -- e.g. to
+// populate QueryAuditEntry's tenant/user once Phase 4's audit wiring
+// threads identity through (see queryapi.AuditLogger's doc comment).
+func IdentityFromContext(ctx context.Context) (Identity, bool) {
+	id, ok := ctx.Value(identityContextKey{}).(Identity)
+	return id, ok
+}
+
+// WithIdentity attaches an already-resolved Identity to ctx -- exported
+// (not just middleware.go's internal use) so packages that construct
+// their own request context outside an HTTP handler -- e.g. enterprise/
+// internal/chrunner's tests, or a future non-HTTP caller -- can put a
+// real Identity in context the same way RequireRole/RequireRoleOrService
+// do, rather than reaching for an unexported field via reflection or
+// duplicating this one-line function.
+func WithIdentity(ctx context.Context, id Identity) context.Context {
+	return context.WithValue(ctx, identityContextKey{}, id)
+}

api/authz/authz_test.go

@@ -0,0 +1,36 @@
+package authz
+
+import "testing"
+
+func TestRoleSatisfies(t *testing.T) {
+	tests := []struct {
+		have, want Role
+		satisfies  bool
+	}{
+		{RoleViewer, RoleViewer, true},
+		{RoleEditor, RoleViewer, true},
+		{RoleAdmin, RoleViewer, true},
+		{RoleOwner, RoleViewer, true},
+		{RoleViewer, RoleEditor, false},
+		{RoleViewer, RoleAdmin, false},
+		{RoleEditor, RoleAdmin, false},
+		{RoleAdmin, RoleOwner, false},
+		{RoleOwner, RoleOwner, true},
+		// RoleService is a separate lane, not on the human rank scale --
+		// per /docs/phase-4-isolation-design.md's alerting service
+		// identity, it must never satisfy a human role requirement no
+		// matter how "high" that might look on paper, and a human role
+		// (even Owner) must never satisfy a RoleService requirement.
+		{RoleService, RoleViewer, false},
+		{RoleService, RoleOwner, false},
+		{RoleOwner, RoleService, false},
+		{RoleViewer, RoleService, false},
+		{RoleService, RoleService, true},
+	}
+	for _, tt := range tests {
+		got := tt.have.Satisfies(tt.want)
+		if got != tt.satisfies {
+			t.Errorf("Role(%q).Satisfies(%q) = %v, want %v", tt.have, tt.want, got, tt.satisfies)
+		}
+	}
+}

api/authz/httpauthz.go

@@ -0,0 +1,65 @@
+package authz
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"time"
+)
+
+// HTTPAuthorizer is the production Authorizer for a deployment with
+// enterprise-auth configured -- it calls enterprise-auth's
+// POST /internal/authorize endpoint, forwarding the caller's own
+// credentials (session cookie or service-token header), rather than
+// importing any enterprise/ Go package. This is the same "network
+// boundary, not import boundary" shape /alerting's queryclient already
+// uses to call api's /query: the module-boundary guarantee
+// hack/check-tenant-boundary.sh enforces is about Go imports, and this
+// type has none from enterprise/.
+type HTTPAuthorizer struct {
+	baseURL string
+	http    *http.Client
+}
+
+func NewHTTPAuthorizer(baseURL string) *HTTPAuthorizer {
+	return &HTTPAuthorizer{baseURL: baseURL, http: &http.Client{Timeout: 3 * time.Second}}
+}
+
+type authorizeResponse struct {
+	TenantID string `json:"tenant_id"`
+	UserID   string `json:"user_id"`
+	Role     string `json:"role"`
+}
+
+// Authorize forwards exactly two credential-carrying headers -- Cookie
+// (human sessions) and Authorization (service tokens) -- never the rest
+// of the request. enterprise-auth validates whichever is present and
+// returns the resolved identity, or a non-2xx if neither validates.
+func (a *HTTPAuthorizer) Authorize(r *http.Request) (Identity, error) {
+	req, err := http.NewRequestWithContext(r.Context(), http.MethodPost, a.baseURL+"/internal/authorize", nil)
+	if err != nil {
+		return Identity{}, fmt.Errorf("authz: building request: %w", err)
+	}
+	if cookie := r.Header.Get("Cookie"); cookie != "" {
+		req.Header.Set("Cookie", cookie)
+	}
+	if auth := r.Header.Get("Authorization"); auth != "" {
+		req.Header.Set("Authorization", auth)
+	}
+
+	resp, err := a.http.Do(req)
+	if err != nil {
+		return Identity{}, fmt.Errorf("authz: calling enterprise-auth: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return Identity{}, fmt.Errorf("authz: enterprise-auth returned status %d", resp.StatusCode)
+	}
+
+	var body authorizeResponse
+	if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
+		return Identity{}, fmt.Errorf("authz: decoding response: %w", err)
+	}
+	return Identity{TenantID: body.TenantID, UserID: body.UserID, Role: Role(body.Role)}, nil
+}

api/authz/httpauthz_test.go

@@ -0,0 +1,72 @@
+package authz
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestHTTPAuthorizerForwardsCredentialsAndParsesIdentity(t *testing.T) {
+	var gotCookie, gotAuth string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotCookie = r.Header.Get("Cookie")
+		gotAuth = r.Header.Get("Authorization")
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(authorizeResponse{TenantID: "acme", UserID: "u1", Role: "editor"})
+	}))
+	defer srv.Close()
+
+	a := NewHTTPAuthorizer(srv.URL)
+	incoming := httptest.NewRequest(http.MethodPost, "/query", nil)
+	incoming.Header.Set("Cookie", "sentry_session=abc123")
+	incoming.Header.Set("Authorization", "Bearer service-token-xyz")
+
+	identity, err := a.Authorize(incoming)
+	if err != nil {
+		t.Fatalf("Authorize: %v", err)
+	}
+	if identity.TenantID != "acme" || identity.UserID != "u1" || identity.Role != RoleEditor {
+		t.Fatalf("unexpected identity: %+v", identity)
+	}
+	if gotCookie != "sentry_session=abc123" {
+		t.Fatalf("Cookie header not forwarded, got %q", gotCookie)
+	}
+	if gotAuth != "Bearer service-token-xyz" {
+		t.Fatalf("Authorization header not forwarded, got %q", gotAuth)
+	}
+}
+
+func TestHTTPAuthorizerNon2xxIsAnError(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusUnauthorized)
+	}))
+	defer srv.Close()
+
+	a := NewHTTPAuthorizer(srv.URL)
+	_, err := a.Authorize(httptest.NewRequest(http.MethodPost, "/query", nil))
+	if err == nil {
+		t.Fatalf("expected an error for a 401 response from enterprise-auth")
+	}
+}
+
+func TestHTTPAuthorizerDoesNotForwardUnrelatedHeaders(t *testing.T) {
+	var gotXForwarded string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotXForwarded = r.Header.Get("X-Forwarded-For")
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(authorizeResponse{Role: "viewer"})
+	}))
+	defer srv.Close()
+
+	a := NewHTTPAuthorizer(srv.URL)
+	incoming := httptest.NewRequest(http.MethodPost, "/query", nil)
+	incoming.Header.Set("X-Forwarded-For", "1.2.3.4")
+
+	if _, err := a.Authorize(incoming); err != nil {
+		t.Fatalf("Authorize: %v", err)
+	}
+	if gotXForwarded != "" {
+		t.Fatalf("expected only Cookie/Authorization to be forwarded, but X-Forwarded-For leaked through as %q", gotXForwarded)
+	}
+}

api/authz/middleware.go

@@ -0,0 +1,72 @@
+package authz
+
+import (
+	"encoding/json"
+	"net/http"
+)
+
+type errorResponse struct {
+	Error string `json:"error"`
+}
+
+func writeUnauthorized(w http.ResponseWriter) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusUnauthorized)
+	_ = json.NewEncoder(w).Encode(errorResponse{Error: "unauthorized"})
+}
+
+func writeForbidden(w http.ResponseWriter) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusForbidden)
+	_ = json.NewEncoder(w).Encode(errorResponse{Error: "forbidden"})
+}
+
+// RequireRole wraps next so it only runs for a caller whose resolved
+// Identity.Role satisfies minRole. A nil authorizer is a deliberate,
+// documented no-op -- a single-tenant deployment with no enterprise/
+// configured behaves exactly as Phases 0-3 did, unauthenticated, not
+// locked out. This is the same nil-safety shape as
+// queryapi.AuditLogger and dashboards' optional dependencies.
+func RequireRole(authorizer Authorizer, minRole Role, next http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if authorizer == nil {
+			next(w, r)
+			return
+		}
+		identity, err := authorizer.Authorize(r)
+		if err != nil {
+			writeUnauthorized(w)
+			return
+		}
+		if !identity.Role.Satisfies(minRole) {
+			writeForbidden(w)
+			return
+		}
+		next(w, r.WithContext(WithIdentity(r.Context(), identity)))
+	}
+}
+
+// RequireRoleOrService is RequireRole plus an explicit allowance for
+// RoleService -- used only by endpoints /alerting's evaluator legitimately
+// calls (POST /query today). Every other endpoint uses plain RequireRole,
+// so a service credential can never reach dashboard/rule administration
+// even though it's a valid, authenticated identity -- narrow by default,
+// widened only where a real machine caller exists.
+func RequireRoleOrService(authorizer Authorizer, minRole Role, next http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if authorizer == nil {
+			next(w, r)
+			return
+		}
+		identity, err := authorizer.Authorize(r)
+		if err != nil {
+			writeUnauthorized(w)
+			return
+		}
+		if identity.Role != RoleService && !identity.Role.Satisfies(minRole) {
+			writeForbidden(w)
+			return
+		}
+		next(w, r.WithContext(WithIdentity(r.Context(), identity)))
+	}
+}

api/authz/middleware_test.go

@@ -0,0 +1,96 @@
+package authz
+
+import (
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+type fakeAuthorizer struct {
+	identity Identity
+	err      error
+}
+
+func (f *fakeAuthorizer) Authorize(_ *http.Request) (Identity, error) {
+	return f.identity, f.err
+}
+
+func okHandler(w http.ResponseWriter, r *http.Request) {
+	id, ok := IdentityFromContext(r.Context())
+	if ok {
+		w.Header().Set("X-Test-Tenant", id.TenantID)
+	}
+	w.WriteHeader(http.StatusOK)
+}
+
+func TestRequireRoleNilAuthorizerIsNoOp(t *testing.T) {
+	h := RequireRole(nil, RoleAdmin, okHandler)
+	rec := httptest.NewRecorder()
+	h(rec, httptest.NewRequest(http.MethodGet, "/", nil))
+	if rec.Code != http.StatusOK {
+		t.Fatalf("status = %d, want 200 (nil authorizer must be a no-op, matching single-tenant Phase 0-3 behavior)", rec.Code)
+	}
+}
+
+func TestRequireRoleRejectsUnauthenticated(t *testing.T) {
+	h := RequireRole(&fakeAuthorizer{err: errors.New("no session")}, RoleViewer, okHandler)
+	rec := httptest.NewRecorder()
+	h(rec, httptest.NewRequest(http.MethodGet, "/", nil))
+	if rec.Code != http.StatusUnauthorized {
+		t.Fatalf("status = %d, want 401", rec.Code)
+	}
+}
+
+func TestRequireRoleRejectsInsufficientRole(t *testing.T) {
+	h := RequireRole(&fakeAuthorizer{identity: Identity{Role: RoleViewer}}, RoleAdmin, okHandler)
+	rec := httptest.NewRecorder()
+	h(rec, httptest.NewRequest(http.MethodGet, "/", nil))
+	if rec.Code != http.StatusForbidden {
+		t.Fatalf("status = %d, want 403", rec.Code)
+	}
+}
+
+func TestRequireRoleAllowsSufficientRoleAndAttachesIdentity(t *testing.T) {
+	h := RequireRole(&fakeAuthorizer{identity: Identity{TenantID: "acme", Role: RoleAdmin}}, RoleEditor, okHandler)
+	rec := httptest.NewRecorder()
+	h(rec, httptest.NewRequest(http.MethodGet, "/", nil))
+	if rec.Code != http.StatusOK {
+		t.Fatalf("status = %d, want 200", rec.Code)
+	}
+	if got := rec.Header().Get("X-Test-Tenant"); got != "acme" {
+		t.Fatalf("expected the resolved identity to be attached to the request context, got tenant=%q", got)
+	}
+}
+
+func TestRequireRoleOrServiceAllowsServiceIdentity(t *testing.T) {
+	h := RequireRoleOrService(&fakeAuthorizer{identity: Identity{Role: RoleService}}, RoleAdmin, okHandler)
+	rec := httptest.NewRecorder()
+	h(rec, httptest.NewRequest(http.MethodGet, "/", nil))
+	if rec.Code != http.StatusOK {
+		t.Fatalf("status = %d, want 200 -- RoleService must satisfy RequireRoleOrService regardless of minRole", rec.Code)
+	}
+}
+
+func TestRequireRoleOrServiceStillRejectsInsufficientHumanRole(t *testing.T) {
+	h := RequireRoleOrService(&fakeAuthorizer{identity: Identity{Role: RoleViewer}}, RoleAdmin, okHandler)
+	rec := httptest.NewRecorder()
+	h(rec, httptest.NewRequest(http.MethodGet, "/", nil))
+	if rec.Code != http.StatusForbidden {
+		t.Fatalf("status = %d, want 403 -- allowing RoleService must not loosen the human-role check", rec.Code)
+	}
+}
+
+// TestRequireRolePlainDoesNotAllowService pins down the narrow-by-default
+// property middleware.go's doc comment claims: an endpoint using plain
+// RequireRole (not RequireRoleOrService) must reject a RoleService
+// identity even if the minRole would technically be satisfiable on the
+// human scale -- RoleService never satisfies anything but itself.
+func TestRequireRolePlainDoesNotAllowService(t *testing.T) {
+	h := RequireRole(&fakeAuthorizer{identity: Identity{Role: RoleService}}, RoleViewer, okHandler)
+	rec := httptest.NewRecorder()
+	h(rec, httptest.NewRequest(http.MethodGet, "/", nil))
+	if rec.Code != http.StatusForbidden {
+		t.Fatalf("status = %d, want 403 -- plain RequireRole must never admit a service identity", rec.Code)
+	}
+}