Close search's active-tenant write-routing gap with a polled allowlist
search/src/consumer.rs's write-routing (built last pass) had no active- tenant check at all: IndexRegistry.resolve() would open-or-create an index directory for any syntactically-valid tenant_id, active or not -- unlike ClickHouse's chwriter.Registry (an active-tenants-only snapshot built at enterprise-ingest startup) or the read side (gated by searchclient.TenantChecker, a direct rbacstore query). search is AGPL core with no Postgres access and no enterprise/ import allowed, so it needed a network boundary instead -- the same shape ingest's TenantResolver already uses against enterprise-auth, just Rust calling Go instead of Go calling Go. New GET /internal/active-tenants endpoint on enterprise-auth (rbacstore.ListActiveTenantIDs + authhandler.handleActiveTenants), gated on a RoleService Bearer credential -- server-to-server auth, the same shape alerting presents to api, minted via the already-generic enterprise-auth -mint-service-token search. search/src/tenants.rs's ActiveTenantTracker polls it every 60s, blocking startup on the first fetch succeeding (fail-closed cold start -- a control-plane outage at boot must not silently accept every tenant_id) and keeping the last- known-good set on any later refresh failure (a transient blip shouldn't stop every tenant's indexing, only prevent the allowlist from growing/ shrinking until connectivity resumes). consumer.rs refuses any tagged record whose tenant isn't in the polled set, before ever calling resolve() -- IndexRegistry itself stays policy-free, matching the same mechanism/policy split clickhousewriter.Writer vs. chwriter.Registry already draws on the ClickHouse side. Off unless ENTERPRISE_AUTH_URL/ENTERPRISE_AUTH_SERVICE_TOKEN are both set (search/src/config.rs rejects exactly one being set) -- every existing deployment is unaffected. Verified with real HTTP round trips in this environment: tenants.rs's tests exercise real reqwest requests (actual Authorization: Bearer header, actual JSON parsing) against a hand-rolled dependency-free TCP test server, including both fail-closed paths (rejected first fetch, unreachable server). authhandler's new tests cover the credential-kind distinction this endpoint exists to enforce -- a real human session, even for a genuine Owner, must not satisfy a check meant for a service identity. One asymmetry remains, disclosed rather than fixed: chwriter.Registry's snapshot still never refreshes (stale until enterprise-ingest restarts), while ActiveTenantTracker's 60s poll gives Tantivy a materially tighter staleness window. Neither is a live per-write check -- that would mean a database/HTTP round trip per record, a throughput cost neither implementation accepts -- so both have some staleness window by design; the gap between the two windows is what's disclosed, not a claim either is fully live.
This commit is contained in:
@@ -36,13 +36,27 @@ type fakeNotFoundError struct{}
|
||||
|
||||
func (*fakeNotFoundError) Error() string { return "not found" }
|
||||
|
||||
// fakeTenantLister is an in-memory stand-in for *rbacstore.Store's
|
||||
// ListActiveTenantIDs.
|
||||
type fakeTenantLister struct {
|
||||
ids []string
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeTenantLister) ListActiveTenantIDs(_ context.Context) ([]string, error) {
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
return f.ids, nil
|
||||
}
|
||||
|
||||
func testHandler(t *testing.T) (*Handler, *session.Manager) {
|
||||
t.Helper()
|
||||
m, err := session.NewManager([]byte("this-is-a-32-byte-test-signing-key!"))
|
||||
if err != nil {
|
||||
t.Fatalf("session.NewManager: %v", err)
|
||||
}
|
||||
return New(slog.New(slog.NewTextHandler(io.Discard, nil)), m, Features{}, newFakeIngestCredentialValidator()), m
|
||||
return New(slog.New(slog.NewTextHandler(io.Discard, nil)), m, Features{}, newFakeIngestCredentialValidator(), &fakeTenantLister{}), m
|
||||
}
|
||||
|
||||
func doAuthorize(t *testing.T, h *Handler, mutate func(*http.Request)) *httptest.ResponseRecorder {
|
||||
@@ -146,7 +160,7 @@ func TestFeaturesReflectsConfiguredMechanisms(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("session.NewManager: %v", err)
|
||||
}
|
||||
h := New(slog.New(slog.NewTextHandler(io.Discard, nil)), m, Features{OIDCEnabled: true, SAMLEnabled: false}, newFakeIngestCredentialValidator())
|
||||
h := New(slog.New(slog.NewTextHandler(io.Discard, nil)), m, Features{OIDCEnabled: true, SAMLEnabled: false}, newFakeIngestCredentialValidator(), &fakeTenantLister{})
|
||||
|
||||
mux := http.NewServeMux()
|
||||
h.RegisterRoutes(mux)
|
||||
@@ -219,7 +233,7 @@ func TestAuthorizeIngestResolvesTenant(t *testing.T) {
|
||||
}
|
||||
validator := newFakeIngestCredentialValidator()
|
||||
validator.tenantByToken["real-token"] = "acme"
|
||||
h := New(slog.New(slog.NewTextHandler(io.Discard, nil)), m, Features{}, validator)
|
||||
h := New(slog.New(slog.NewTextHandler(io.Discard, nil)), m, Features{}, validator, &fakeTenantLister{})
|
||||
|
||||
rec := doAuthorizeIngest(t, h, func(r *http.Request) {
|
||||
r.Header.Set("Authorization", "Bearer real-token")
|
||||
@@ -273,3 +287,96 @@ func TestAuthorizeIngestRejectsSessionToken(t *testing.T) {
|
||||
t.Fatalf("status = %d, want 401 (a session token must not validate as an ingest credential)", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func doActiveTenants(t *testing.T, h *Handler, mutate func(*http.Request)) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
mux := http.NewServeMux()
|
||||
h.RegisterRoutes(mux)
|
||||
req := httptest.NewRequest(http.MethodGet, "/internal/active-tenants", nil)
|
||||
if mutate != nil {
|
||||
mutate(req)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
mux.ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
func TestActiveTenantsViaServiceToken(t *testing.T) {
|
||||
m, err := session.NewManager([]byte("this-is-a-32-byte-test-signing-key!"))
|
||||
if err != nil {
|
||||
t.Fatalf("session.NewManager: %v", err)
|
||||
}
|
||||
h := New(slog.New(slog.NewTextHandler(io.Discard, nil)), m, Features{}, newFakeIngestCredentialValidator(), &fakeTenantLister{ids: []string{"acme", "globex"}})
|
||||
token, err := m.IssueServiceToken("search")
|
||||
if err != nil {
|
||||
t.Fatalf("IssueServiceToken: %v", err)
|
||||
}
|
||||
|
||||
rec := doActiveTenants(t, h, func(r *http.Request) {
|
||||
r.Header.Set("Authorization", "Bearer "+token)
|
||||
})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var body activeTenantsResponse
|
||||
if err := json.NewDecoder(rec.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decoding response: %v", err)
|
||||
}
|
||||
if len(body.TenantIDs) != 2 || body.TenantIDs[0] != "acme" || body.TenantIDs[1] != "globex" {
|
||||
t.Fatalf("unexpected response: %+v", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestActiveTenantsNoCredentialsIsUnauthorized(t *testing.T) {
|
||||
h, _ := testHandler(t)
|
||||
rec := doActiveTenants(t, h, nil)
|
||||
if rec.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("status = %d, want 401", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// TestActiveTenantsRejectsHumanSession is the regression test for this
|
||||
// endpoint's whole reason to distinguish token kinds: a human session
|
||||
// (even a real, validly-signed one) must not be able to list every
|
||||
// active tenant in the deployment -- only a RoleService credential can.
|
||||
func TestActiveTenantsRejectsHumanSession(t *testing.T) {
|
||||
h, m := testHandler(t)
|
||||
sessionToken, err := m.IssueUserSession("acme", "u1", "owner")
|
||||
if err != nil {
|
||||
t.Fatalf("IssueUserSession: %v", err)
|
||||
}
|
||||
rec := doActiveTenants(t, h, func(r *http.Request) {
|
||||
r.Header.Set("Authorization", "Bearer "+sessionToken)
|
||||
})
|
||||
if rec.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("status = %d, want 401 (a human session must not satisfy the service-only active-tenants endpoint)", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestActiveTenantsInvalidTokenIsUnauthorized(t *testing.T) {
|
||||
h, _ := testHandler(t)
|
||||
rec := doActiveTenants(t, h, func(r *http.Request) {
|
||||
r.Header.Set("Authorization", "Bearer not-a-real-token")
|
||||
})
|
||||
if rec.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("status = %d, want 401", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestActiveTenantsStoreErrorIsInternalError(t *testing.T) {
|
||||
m, err := session.NewManager([]byte("this-is-a-32-byte-test-signing-key!"))
|
||||
if err != nil {
|
||||
t.Fatalf("session.NewManager: %v", err)
|
||||
}
|
||||
h := New(slog.New(slog.NewTextHandler(io.Discard, nil)), m, Features{}, newFakeIngestCredentialValidator(), &fakeTenantLister{err: errNotFound})
|
||||
token, err := m.IssueServiceToken("search")
|
||||
if err != nil {
|
||||
t.Fatalf("IssueServiceToken: %v", err)
|
||||
}
|
||||
rec := doActiveTenants(t, h, func(r *http.Request) {
|
||||
r.Header.Set("Authorization", "Bearer "+token)
|
||||
})
|
||||
if rec.Code != http.StatusInternalServerError {
|
||||
t.Fatalf("status = %d, want 500", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user