Give ingest a real tenant identity (write-routing deferred, disclosed)

Ingest tenant-awareness was named "undesigned, not just unbuilt" across
CLAUDE.md/threat-model.md/the runbook since early Phase 4 -- the last
major standing gap. Scoping was agreed via AskUserQuestion: a
config-supplied tenant_id + shared-secret token ingest validates
(smaller real implementation, no new PKI), over per-tenant mTLS
certs. This change builds that identity mechanism end to end and
attaches it to every record at the point it enters the system; it
deliberately does NOT build per-tenant write-routing for ClickHouse or
Tantivy -- that's real, separately-scoped follow-up work, disclosed
explicitly everywhere this was previously called undesigned, not
silently left half-done.

New pieces:

- metadata/migrations/0034 + enterprise/internal/rbacstore/
  ingest_credentials.go: a per-tenant bearer credential, only its
  SHA-256 hash ever persisted (same reasoning a password gets hashed,
  not stored raw) -- CreateIngestCredential returns the plaintext
  exactly once, ValidateIngestCredential/RevokeIngestCredential/
  ListIngestCredentialsForTenant round it out.
- enterprise-auth gains -create-ingest-credential-tenant/
  -list-ingest-credentials-tenant/-revoke-ingest-credential (same
  offline-operator-flag shape as every other credential-minting flag in
  this binary) and a new POST /internal/authorize-ingest endpoint
  (internal/authhandler) validating a presented token and resolving its
  tenant -- a genuinely different credential type from session-backed
  /internal/authorize, so it doesn't touch session.Manager at all.
- ingest (AGPL core) gains an optional TenantResolver
  (internal/grpcserver, nil by default) and its HTTP client
  implementation (internal/tenantresolver.HTTPResolver) -- a plain HTTP
  call to enterprise-auth's new endpoint, never an enterprise/ import,
  same "network boundary, not import boundary" shape
  api/authz.HTTPAuthorizer already uses for the query path.
  PushBatch now requires an `authorization: Bearer <token>` gRPC
  metadata entry once a resolver is configured, fails the whole batch
  closed on a missing/invalid credential (never falls back to "no
  tenant"), and attaches the resolved tenant ID to every record as a
  `tenant_id` Kafka message header before producing it.

Verified with real round trips at every layer, no Docker needed:
rbacstore's credential CRUD (skip-gated on live Postgres, same as every
other rbacstore integration test this phase), authhandler's new
endpoint (real HTTP via httptest, including the regression test that a
session token must not validate as an ingest credential), tenantresolver
(real HTTP client against httptest, same pattern as
authz.HTTPAuthorizer's own tests), and grpcserver's PushBatch (fake
resolver/producer -- no resolver leaves messages unchanged, a configured
resolver attaches the right header or fails closed on a bad/missing
token).

Helm: ingest.requireTenantCredential (default false) is a deliberate,
separate opt-in from enterprise.enabled -- turning ENTERPRISE_AUTH_URL
on for ingest requires every agent to already hold a credential or be
refused outright, so it must not default on just because
enterprise.enabled does (same reasoning api.yaml's ENTERPRISE_AUTH_URL
isn't tied to enterprise.enabled directly either). docker-compose.yml
leaves it unset, same as ever.

Docs updated everywhere this was called "undesigned": CLAUDE.md,
docs/architecture.md, docs/security/threat-model.md (including its
summary table, now split into "identity: built" vs "write-routing: not
yet"), docs/phase-4-runbook.md (new §13), enterprise/README.md.
This commit is contained in:
2026-08-14 15:21:55 -07:00
parent d2c76aa3a4
commit 17fdc212c2
21 changed files with 1071 additions and 68 deletions
+7
View File
@@ -17,6 +17,12 @@ type Config struct {
Redpanda RedpandaConfig
ClickHouse ClickHouseConfig
Batch BatchConfig
// EnterpriseAuthURL enables per-tenant ingest credential validation
// (internal/grpcserver.TenantResolver) when set -- empty (the
// default) is a documented no-op, same "off unless configured" shape
// as every other optional enterprise integration point in this
// codebase (e.g. api's own ENTERPRISE_AUTH_URL).
EnterpriseAuthURL string
}
type GRPCConfig struct {
@@ -70,6 +76,7 @@ func Load() (Config, error) {
Username: getenv("CLICKHOUSE_USERNAME", "default"),
Password: getenv("CLICKHOUSE_PASSWORD", ""),
},
EnterpriseAuthURL: getenv("ENTERPRISE_AUTH_URL", ""),
}
maxSize, err := strconv.Atoi(getenv("CONSUMER_BATCH_MAX_SIZE", "500"))
+7
View File
@@ -19,12 +19,16 @@ func TestLoadDefaults(t *testing.T) {
if cfg.Batch.FlushIntervalMS != 2000 {
t.Errorf("Batch.FlushIntervalMS = %d, want 2000", cfg.Batch.FlushIntervalMS)
}
if cfg.EnterpriseAuthURL != "" {
t.Errorf("EnterpriseAuthURL = %q, want empty (tenant resolution off by default)", cfg.EnterpriseAuthURL)
}
}
func TestLoadOverridesFromEnv(t *testing.T) {
t.Setenv("GRPC_LISTEN_ADDR", ":9999")
t.Setenv("REDPANDA_BROKERS", "a:9092,b:9092")
t.Setenv("CONSUMER_BATCH_MAX_SIZE", "10")
t.Setenv("ENTERPRISE_AUTH_URL", "http://enterprise-auth:8082")
cfg, err := Load()
if err != nil {
@@ -39,6 +43,9 @@ func TestLoadOverridesFromEnv(t *testing.T) {
if cfg.Batch.MaxSize != 10 {
t.Errorf("Batch.MaxSize = %d, want 10", cfg.Batch.MaxSize)
}
if cfg.EnterpriseAuthURL != "http://enterprise-auth:8082" {
t.Errorf("EnterpriseAuthURL = %q, want http://enterprise-auth:8082", cfg.EnterpriseAuthURL)
}
}
func TestLoadInvalidBatchSizeErrors(t *testing.T) {
+90 -5
View File
@@ -4,6 +4,22 @@
// happen exactly once, here, rather than in either downstream consumer)
// and otherwise forwards records unchanged onto Redpanda — normalization
// into the ClickHouse row shape happens later, on the consumer side.
//
// If a TenantResolver is configured, PushBatch also resolves which
// tenant the call's bearer credential belongs to and attaches it as a
// "tenant_id" Kafka message header on every record produced -- the first
// step of Phase 4's ingest tenant-awareness (see
// /docs/phase-4-runbook.md and CLAUDE.md's "ingest itself has no tenant
// concept" gap). Deliberately scoped no further than that for now:
// nothing downstream (this package's own consumer, or `search`'s
// separate Redpanda consumer) reads that header yet to route a record's
// write into a per-tenant ClickHouse database/Tantivy index -- every
// record still lands in the one shared destination either way, tenant_id
// header or not. That's real, disclosed, deferred follow-up work, not
// silently incomplete: attaching a verifiable tenant identity as early
// as possible (right where the credential is actually presented) is a
// self-contained, independently valuable step on its own, and it's what
// any later per-tenant write-routing work will consume.
package grpcserver
import (
@@ -11,12 +27,14 @@ import (
"fmt"
"log/slog"
"net"
"strings"
"github.com/google/uuid"
"github.com/segmentio/kafka-go"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/proto"
@@ -24,6 +42,12 @@ import (
logsv1 "github.com/sentry/sentry/proto/sentry/logs/v1"
)
// TenantIDHeaderKey is the Kafka message header a resolved tenant ID is
// attached under -- exported so internal/consumer (or a future per-
// tenant write-routing consumer) can read it back by the same name
// without duplicating the literal.
const TenantIDHeaderKey = "tenant_id"
type Server struct {
logsv1.UnimplementedLogIngestServer
@@ -31,6 +55,7 @@ type Server struct {
grpcCfg config.GRPCConfig
tlsCfg config.TLSConfig
producer batchProducer
resolver TenantResolver
}
// batchProducer is the subset of *producer.Producer this package depends
@@ -39,8 +64,22 @@ type batchProducer interface {
WriteBatch(ctx context.Context, msgs []kafka.Message) error
}
func New(logger *slog.Logger, grpcCfg config.GRPCConfig, tlsCfg config.TLSConfig, p batchProducer) *Server {
return &Server{logger: logger, grpcCfg: grpcCfg, tlsCfg: tlsCfg, producer: p}
// TenantResolver validates an ingest credential (a bearer token
// presented via gRPC metadata, `authorization: Bearer <token>`) and
// resolves which tenant it belongs to. nil is a deliberate no-op: every
// record's Kafka message gets no tenant_id header at all, matching every
// ingest deployment's behavior before per-tenant ingest credentials
// existed. The real implementation
// (ingest/internal/tenantresolver.HTTPResolver) is a plain HTTP client
// calling enterprise-auth's /internal/authorize-ingest -- never an
// enterprise/ import, since this package is AGPL core (same "network
// boundary, not import boundary" shape api/authz.Authorizer uses).
type TenantResolver interface {
ResolveTenant(ctx context.Context, token string) (tenantID string, err error)
}
func New(logger *slog.Logger, grpcCfg config.GRPCConfig, tlsCfg config.TLSConfig, p batchProducer, resolver TenantResolver) *Server {
return &Server{logger: logger, grpcCfg: grpcCfg, tlsCfg: tlsCfg, producer: p, resolver: resolver}
}
// Run blocks serving gRPC until ctx is canceled, then gracefully stops.
@@ -77,6 +116,28 @@ func (s *Server) PushBatch(ctx context.Context, req *logsv1.PushBatchRequest) (*
return &logsv1.PushBatchResponse{Accepted: 0}, nil
}
// tenantID stays empty (no header attached below) unless a resolver
// is actually configured -- single-tenant deployments never present
// a bearer credential and never need to. Once a resolver IS
// configured, a missing/invalid credential fails the whole batch
// closed rather than falling back to "no tenant" -- exactly the
// same fail-closed shape enterprise/internal/chrunner.Registry.RunSQL
// uses on the read side, applied here at the point data enters the
// system.
var tenantID string
if s.resolver != nil {
token, ok := bearerTokenFromContext(ctx)
if !ok {
return nil, status.Error(codes.Unauthenticated, "missing bearer credential")
}
resolved, err := s.resolver.ResolveTenant(ctx, token)
if err != nil {
s.logger.Error("resolving ingest tenant", "batch_id", req.GetBatchId(), "error", err)
return nil, status.Error(codes.Unauthenticated, "invalid ingest credential")
}
tenantID = resolved
}
msgs := make([]kafka.Message, 0, len(req.GetRecords()))
for _, rec := range req.GetRecords() {
// Assigned here, once, before this record is produced to
@@ -92,10 +153,14 @@ func (s *Server) PushBatch(ctx context.Context, req *logsv1.PushBatchRequest) (*
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "marshaling record: %v", err)
}
msgs = append(msgs, kafka.Message{
msg := kafka.Message{
Key: []byte(rec.GetHost()),
Value: val,
})
}
if tenantID != "" {
msg.Headers = []kafka.Header{{Key: TenantIDHeaderKey, Value: []byte(tenantID)}}
}
msgs = append(msgs, msg)
}
if err := s.producer.WriteBatch(ctx, msgs); err != nil {
@@ -103,6 +168,26 @@ func (s *Server) PushBatch(ctx context.Context, req *logsv1.PushBatchRequest) (*
return nil, status.Errorf(codes.Unavailable, "writing to transport: %v", err)
}
s.logger.Debug("batch produced to redpanda", "batch_id", req.GetBatchId(), "records", len(req.GetRecords()))
s.logger.Debug("batch produced to redpanda", "batch_id", req.GetBatchId(), "records", len(req.GetRecords()), "tenant_id", tenantID)
return &logsv1.PushBatchResponse{Accepted: uint32(len(req.GetRecords()))}, nil
}
// bearerTokenFromContext reads the same "authorization: Bearer <token>"
// gRPC metadata shape HTTP's Authorization header uses -- an agent sets
// this once per PushBatch call (see the agent's grpc.rs), not per
// record.
func bearerTokenFromContext(ctx context.Context) (string, bool) {
md, ok := metadata.FromIncomingContext(ctx)
if !ok {
return "", false
}
values := md.Get("authorization")
if len(values) == 0 {
return "", false
}
const prefix = "Bearer "
if !strings.HasPrefix(values[0], prefix) {
return "", false
}
return strings.TrimPrefix(values[0], prefix), true
}
+130 -1
View File
@@ -2,12 +2,16 @@ package grpcserver
import (
"context"
"errors"
"io"
"log/slog"
"sync"
"testing"
"github.com/segmentio/kafka-go"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/proto"
"github.com/sentry/sentry/ingest/internal/config"
@@ -32,8 +36,35 @@ func (f *fakeProducer) WriteBatch(_ context.Context, msgs []kafka.Message) error
return nil
}
// fakeResolver is an in-memory stand-in for
// ingest/internal/tenantresolver.HTTPResolver, keyed by token.
type fakeResolver struct {
tenantByToken map[string]string
}
func (f *fakeResolver) ResolveTenant(_ context.Context, token string) (string, error) {
tenantID, ok := f.tenantByToken[token]
if !ok {
return "", errors.New("fakeResolver: unknown token")
}
return tenantID, nil
}
func newTestServer(p batchProducer) *Server {
return New(slog.New(slog.NewTextHandler(io.Discard, nil)), config.GRPCConfig{}, config.TLSConfig{}, p)
return New(slog.New(slog.NewTextHandler(io.Discard, nil)), config.GRPCConfig{}, config.TLSConfig{}, p, nil)
}
func newTestServerWithResolver(p batchProducer, resolver TenantResolver) *Server {
return New(slog.New(slog.NewTextHandler(io.Discard, nil)), config.GRPCConfig{}, config.TLSConfig{}, p, resolver)
}
// contextWithBearerToken builds an incoming gRPC context carrying an
// "authorization: Bearer <token>" metadata entry -- the shape a real
// grpc-go server hands PushBatch once TLS/framing is stripped away, so
// this exercises the same metadata.FromIncomingContext path production
// traffic does, not a shortcut around it.
func contextWithBearerToken(token string) context.Context {
return metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "Bearer "+token))
}
func TestPushBatchAssignsRecordID(t *testing.T) {
@@ -124,3 +155,101 @@ func TestPushBatchEmptyRecordsIsANoOp(t *testing.T) {
t.Fatalf("expected no batches written for an empty request, got %d", len(fp.written))
}
}
// TestPushBatchNoResolverAttachesNoTenantHeader is the regression test
// for single-tenant deployments' behavior staying unchanged: with no
// TenantResolver configured, records are produced exactly as before --
// no tenant_id header at all -- even with a bearer token present (it's
// simply never inspected).
func TestPushBatchNoResolverAttachesNoTenantHeader(t *testing.T) {
fp := &fakeProducer{}
s := newTestServer(fp)
req := &logsv1.PushBatchRequest{Records: []*logsv1.LogRecord{{Host: "h1", Message: "one"}}}
if _, err := s.PushBatch(contextWithBearerToken("irrelevant"), req); err != nil {
t.Fatalf("PushBatch() error = %v", err)
}
fp.mu.Lock()
defer fp.mu.Unlock()
for _, h := range fp.written[0][0].Headers {
if h.Key == TenantIDHeaderKey {
t.Fatalf("expected no %s header with no resolver configured, got %q", TenantIDHeaderKey, h.Value)
}
}
}
func TestPushBatchWithResolverAttachesTenantHeader(t *testing.T) {
fp := &fakeProducer{}
resolver := &fakeResolver{tenantByToken: map[string]string{"real-token": "acme"}}
s := newTestServerWithResolver(fp, resolver)
req := &logsv1.PushBatchRequest{Records: []*logsv1.LogRecord{
{Host: "h1", Message: "one"},
{Host: "h1", Message: "two"},
}}
if _, err := s.PushBatch(contextWithBearerToken("real-token"), req); err != nil {
t.Fatalf("PushBatch() error = %v", err)
}
fp.mu.Lock()
defer fp.mu.Unlock()
if len(fp.written[0]) != 2 {
t.Fatalf("expected 2 messages written, got %d", len(fp.written[0]))
}
for _, msg := range fp.written[0] {
found := false
for _, h := range msg.Headers {
if h.Key == TenantIDHeaderKey {
found = true
if string(h.Value) != "acme" {
t.Fatalf("%s header = %q, want acme", TenantIDHeaderKey, h.Value)
}
}
}
if !found {
t.Fatalf("expected every record to carry a %s header", TenantIDHeaderKey)
}
}
}
func TestPushBatchWithResolverRejectsMissingToken(t *testing.T) {
fp := &fakeProducer{}
resolver := &fakeResolver{tenantByToken: map[string]string{"real-token": "acme"}}
s := newTestServerWithResolver(fp, resolver)
req := &logsv1.PushBatchRequest{Records: []*logsv1.LogRecord{{Host: "h1", Message: "one"}}}
_, err := s.PushBatch(context.Background(), req) // no bearer token in context at all
if status.Code(err) != codes.Unauthenticated {
t.Fatalf("PushBatch() error = %v, want Unauthenticated", err)
}
fp.mu.Lock()
defer fp.mu.Unlock()
if len(fp.written) != 0 {
t.Fatal("a batch with no bearer token must never reach the producer once a resolver is configured")
}
}
// TestPushBatchWithResolverRejectsInvalidToken is the fail-closed
// regression test: a resolver configured but a token it doesn't
// recognize must refuse the whole batch, never fall back to "no tenant"
// (which would silently defeat the point of requiring a credential at
// all).
func TestPushBatchWithResolverRejectsInvalidToken(t *testing.T) {
fp := &fakeProducer{}
resolver := &fakeResolver{tenantByToken: map[string]string{"real-token": "acme"}}
s := newTestServerWithResolver(fp, resolver)
req := &logsv1.PushBatchRequest{Records: []*logsv1.LogRecord{{Host: "h1", Message: "one"}}}
_, err := s.PushBatch(contextWithBearerToken("wrong-token"), req)
if status.Code(err) != codes.Unauthenticated {
t.Fatalf("PushBatch() error = %v, want Unauthenticated", err)
}
fp.mu.Lock()
defer fp.mu.Unlock()
if len(fp.written) != 0 {
t.Fatal("a batch with an invalid token must never reach the producer once a resolver is configured")
}
}
@@ -0,0 +1,65 @@
// Package tenantresolver is ingest's HTTP client for resolving an
// agent-presented ingest credential to a tenant -- calls enterprise-
// auth's POST /internal/authorize-ingest over the network, never
// importing enterprise/ (ingest is AGPL core; enterprise/ is
// commercial-licensed and must never be imported by core code -- same
// "network boundary, not import boundary" shape api/authz.HTTPAuthorizer
// already uses for the query path, and enterprise-auth's own doc
// comment on POST /internal/authorize-ingest). nil (no resolver
// configured) is grpcserver.Server's documented no-op default --
// single-tenant deployments never construct one, and every record's
// TenantID stays empty, exactly like before per-tenant ingest
// credentials existed.
package tenantresolver
import (
"context"
"encoding/json"
"fmt"
"net/http"
"time"
)
type HTTPResolver struct {
baseURL string
http *http.Client
}
func New(baseURL string) *HTTPResolver {
return &HTTPResolver{baseURL: baseURL, http: &http.Client{Timeout: 3 * time.Second}}
}
type authorizeIngestResponse struct {
TenantID string `json:"tenant_id"`
}
// ResolveTenant implements grpcserver.TenantResolver. Forwards only the
// bearer token itself, nothing else about the caller's request -- same
// "forward exactly the credential, never the rest of the request"
// discipline api/authz.HTTPAuthorizer already follows.
func (r *HTTPResolver) ResolveTenant(ctx context.Context, token string) (string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.baseURL+"/internal/authorize-ingest", nil)
if err != nil {
return "", fmt.Errorf("tenantresolver: building request: %w", err)
}
req.Header.Set("Authorization", "Bearer "+token)
resp, err := r.http.Do(req)
if err != nil {
return "", fmt.Errorf("tenantresolver: calling enterprise-auth: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("tenantresolver: enterprise-auth returned status %d", resp.StatusCode)
}
var body authorizeIngestResponse
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
return "", fmt.Errorf("tenantresolver: decoding response: %w", err)
}
if body.TenantID == "" {
return "", fmt.Errorf("tenantresolver: enterprise-auth returned an empty tenant_id")
}
return body.TenantID, nil
}
@@ -0,0 +1,56 @@
package tenantresolver
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
)
func TestResolveTenantForwardsTokenAndParsesTenantID(t *testing.T) {
var gotAuth string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotAuth = r.Header.Get("Authorization")
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(authorizeIngestResponse{TenantID: "acme"})
}))
defer srv.Close()
res := New(srv.URL)
tenantID, err := res.ResolveTenant(context.Background(), "real-token")
if err != nil {
t.Fatalf("ResolveTenant: %v", err)
}
if tenantID != "acme" {
t.Fatalf("tenantID = %q, want acme", tenantID)
}
if gotAuth != "Bearer real-token" {
t.Fatalf("Authorization header = %q, want Bearer real-token", gotAuth)
}
}
func TestResolveTenantNon2xxIsAnError(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusUnauthorized)
}))
defer srv.Close()
res := New(srv.URL)
if _, err := res.ResolveTenant(context.Background(), "bad-token"); err == nil {
t.Fatal("expected an error for a 401 response from enterprise-auth")
}
}
func TestResolveTenantRejectsEmptyTenantID(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(authorizeIngestResponse{})
}))
defer srv.Close()
res := New(srv.URL)
if _, err := res.ResolveTenant(context.Background(), "some-token"); err == nil {
t.Fatal("expected an error when enterprise-auth returns an empty tenant_id despite a 200")
}
}