Give ingest a real tenant identity (write-routing deferred, disclosed)

Ingest tenant-awareness was named "undesigned, not just unbuilt" across
CLAUDE.md/threat-model.md/the runbook since early Phase 4 -- the last
major standing gap. Scoping was agreed via AskUserQuestion: a
config-supplied tenant_id + shared-secret token ingest validates
(smaller real implementation, no new PKI), over per-tenant mTLS
certs. This change builds that identity mechanism end to end and
attaches it to every record at the point it enters the system; it
deliberately does NOT build per-tenant write-routing for ClickHouse or
Tantivy -- that's real, separately-scoped follow-up work, disclosed
explicitly everywhere this was previously called undesigned, not
silently left half-done.

New pieces:

- metadata/migrations/0034 + enterprise/internal/rbacstore/
  ingest_credentials.go: a per-tenant bearer credential, only its
  SHA-256 hash ever persisted (same reasoning a password gets hashed,
  not stored raw) -- CreateIngestCredential returns the plaintext
  exactly once, ValidateIngestCredential/RevokeIngestCredential/
  ListIngestCredentialsForTenant round it out.
- enterprise-auth gains -create-ingest-credential-tenant/
  -list-ingest-credentials-tenant/-revoke-ingest-credential (same
  offline-operator-flag shape as every other credential-minting flag in
  this binary) and a new POST /internal/authorize-ingest endpoint
  (internal/authhandler) validating a presented token and resolving its
  tenant -- a genuinely different credential type from session-backed
  /internal/authorize, so it doesn't touch session.Manager at all.
- ingest (AGPL core) gains an optional TenantResolver
  (internal/grpcserver, nil by default) and its HTTP client
  implementation (internal/tenantresolver.HTTPResolver) -- a plain HTTP
  call to enterprise-auth's new endpoint, never an enterprise/ import,
  same "network boundary, not import boundary" shape
  api/authz.HTTPAuthorizer already uses for the query path.
  PushBatch now requires an `authorization: Bearer <token>` gRPC
  metadata entry once a resolver is configured, fails the whole batch
  closed on a missing/invalid credential (never falls back to "no
  tenant"), and attaches the resolved tenant ID to every record as a
  `tenant_id` Kafka message header before producing it.

Verified with real round trips at every layer, no Docker needed:
rbacstore's credential CRUD (skip-gated on live Postgres, same as every
other rbacstore integration test this phase), authhandler's new
endpoint (real HTTP via httptest, including the regression test that a
session token must not validate as an ingest credential), tenantresolver
(real HTTP client against httptest, same pattern as
authz.HTTPAuthorizer's own tests), and grpcserver's PushBatch (fake
resolver/producer -- no resolver leaves messages unchanged, a configured
resolver attaches the right header or fails closed on a bad/missing
token).

Helm: ingest.requireTenantCredential (default false) is a deliberate,
separate opt-in from enterprise.enabled -- turning ENTERPRISE_AUTH_URL
on for ingest requires every agent to already hold a credential or be
refused outright, so it must not default on just because
enterprise.enabled does (same reasoning api.yaml's ENTERPRISE_AUTH_URL
isn't tied to enterprise.enabled directly either). docker-compose.yml
leaves it unset, same as ever.

Docs updated everywhere this was called "undesigned": CLAUDE.md,
docs/architecture.md, docs/security/threat-model.md (including its
summary table, now split into "identity: built" vs "write-routing: not
yet"), docs/phase-4-runbook.md (new §13), enterprise/README.md.

enterprise/internal/authhandler/authhandler.go

@@ -7,9 +7,16 @@
 // signed tokens with a different Role claim, so one validation path
 // handles both, and the Role claim (not which header carried it) is what
 // determines whether the result looks like a human or a service identity.
+//
+// POST /internal/authorize-ingest is a sibling endpoint, same network-
+// boundary shape but for a different caller (`ingest`, core/AGPL, not
+// api/authz) and a different credential type (an ingest bearer token
+// checked against rbacstore's ingest_credentials table, not a
+// session.Manager JWT) -- see ingestCredentialValidator's doc comment.
 package authhandler
 
 import (
+	"context"
 	"encoding/json"
 	"log/slog"
 	"net/http"
@@ -32,19 +39,32 @@ type Features struct {
 	SAMLEnabled bool
 }
 
-type Handler struct {
-	logger   *slog.Logger
-	manager  *session.Manager
-	features Features
+// ingestCredentialValidator is the narrow interface POST
+// /internal/authorize-ingest needs -- *rbacstore.Store is the production
+// implementation. Unlike session-backed /internal/authorize, this
+// endpoint validates a completely different credential type (an ingest
+// bearer token, checked against enterprise/internal/rbacstore's
+// ingest_credentials table, never a session.Manager-signed JWT), so it
+// needs a dependency session.Manager alone can't supply.
+type ingestCredentialValidator interface {
+	ValidateIngestCredential(ctx context.Context, token string) (tenantID string, err error)
 }
 
-func New(logger *slog.Logger, manager *session.Manager, features Features) *Handler {
-	return &Handler{logger: logger, manager: manager, features: features}
+type Handler struct {
+	logger            *slog.Logger
+	manager           *session.Manager
+	features          Features
+	ingestCredentials ingestCredentialValidator
+}
+
+func New(logger *slog.Logger, manager *session.Manager, features Features, ingestCredentials ingestCredentialValidator) *Handler {
+	return &Handler{logger: logger, manager: manager, features: features, ingestCredentials: ingestCredentials}
 }
 
 func (h *Handler) RegisterRoutes(mux *http.ServeMux) {
 	mux.HandleFunc("POST /internal/authorize", h.handleAuthorize)
 	mux.HandleFunc("GET /auth/features", h.handleFeatures)
+	mux.HandleFunc("POST /internal/authorize-ingest", h.handleAuthorizeIngest)
 }
 
 type featuresResponse struct {
@@ -100,6 +120,32 @@ func (h *Handler) handleAuthorize(w http.ResponseWriter, r *http.Request) {
 	})
 }
 
+type authorizeIngestResponse struct {
+	TenantID string `json:"tenant_id"`
+}
+
+// handleAuthorizeIngest is ingest/internal/grpcserver.HTTPTenantResolver's
+// server side -- ingest calls this once per PushBatch (with the bearer
+// token the agent presented) to resolve which tenant the batch belongs
+// to, the network-boundary equivalent of api/authz.HTTPAuthorizer
+// calling /internal/authorize, for a different credential type.
+func (h *Handler) handleAuthorizeIngest(w http.ResponseWriter, r *http.Request) {
+	token := bearerToken(r.Header.Get("Authorization"))
+	if token == "" {
+		http.Error(w, "no credentials presented", http.StatusUnauthorized)
+		return
+	}
+
+	tenantID, err := h.ingestCredentials.ValidateIngestCredential(r.Context(), token)
+	if err != nil {
+		http.Error(w, "invalid ingest credential", http.StatusUnauthorized)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(authorizeIngestResponse{TenantID: tenantID})
+}
+
 func bearerToken(header string) string {
 	const prefix = "Bearer "
 	if !strings.HasPrefix(header, prefix) {

enterprise/internal/authhandler/authhandler_test.go

@@ -1,6 +1,7 @@
 package authhandler
 
 import (
+	"context"
 	"encoding/json"
 	"io"
 	"log/slog"
@@ -11,13 +12,37 @@ import (
 	"github.com/sentry/sentry/enterprise/internal/session"
 )
 
+// fakeIngestCredentialValidator is an in-memory stand-in for
+// *rbacstore.Store's ValidateIngestCredential, keyed by token.
+type fakeIngestCredentialValidator struct {
+	tenantByToken map[string]string
+}
+
+func newFakeIngestCredentialValidator() *fakeIngestCredentialValidator {
+	return &fakeIngestCredentialValidator{tenantByToken: map[string]string{}}
+}
+
+func (f *fakeIngestCredentialValidator) ValidateIngestCredential(_ context.Context, token string) (string, error) {
+	tenantID, ok := f.tenantByToken[token]
+	if !ok {
+		return "", errNotFound
+	}
+	return tenantID, nil
+}
+
+var errNotFound = &fakeNotFoundError{}
+
+type fakeNotFoundError struct{}
+
+func (*fakeNotFoundError) Error() string { return "not found" }
+
 func testHandler(t *testing.T) (*Handler, *session.Manager) {
 	t.Helper()
 	m, err := session.NewManager([]byte("this-is-a-32-byte-test-signing-key!"))
 	if err != nil {
 		t.Fatalf("session.NewManager: %v", err)
 	}
-	return New(slog.New(slog.NewTextHandler(io.Discard, nil)), m, Features{}), m
+	return New(slog.New(slog.NewTextHandler(io.Discard, nil)), m, Features{}, newFakeIngestCredentialValidator()), m
 }
 
 func doAuthorize(t *testing.T, h *Handler, mutate func(*http.Request)) *httptest.ResponseRecorder {
@@ -121,7 +146,7 @@ func TestFeaturesReflectsConfiguredMechanisms(t *testing.T) {
 	if err != nil {
 		t.Fatalf("session.NewManager: %v", err)
 	}
-	h := New(slog.New(slog.NewTextHandler(io.Discard, nil)), m, Features{OIDCEnabled: true, SAMLEnabled: false})
+	h := New(slog.New(slog.NewTextHandler(io.Discard, nil)), m, Features{OIDCEnabled: true, SAMLEnabled: false}, newFakeIngestCredentialValidator())
 
 	mux := http.NewServeMux()
 	h.RegisterRoutes(mux)
@@ -173,3 +198,78 @@ func TestAuthorizeTokenFromWrongManagerIsUnauthorized(t *testing.T) {
 		t.Fatalf("status = %d, want 401", rec.Code)
 	}
 }
+
+func doAuthorizeIngest(t *testing.T, h *Handler, mutate func(*http.Request)) *httptest.ResponseRecorder {
+	t.Helper()
+	mux := http.NewServeMux()
+	h.RegisterRoutes(mux)
+	req := httptest.NewRequest(http.MethodPost, "/internal/authorize-ingest", nil)
+	if mutate != nil {
+		mutate(req)
+	}
+	rec := httptest.NewRecorder()
+	mux.ServeHTTP(rec, req)
+	return rec
+}
+
+func TestAuthorizeIngestResolvesTenant(t *testing.T) {
+	m, err := session.NewManager([]byte("this-is-a-32-byte-test-signing-key!"))
+	if err != nil {
+		t.Fatalf("session.NewManager: %v", err)
+	}
+	validator := newFakeIngestCredentialValidator()
+	validator.tenantByToken["real-token"] = "acme"
+	h := New(slog.New(slog.NewTextHandler(io.Discard, nil)), m, Features{}, validator)
+
+	rec := doAuthorizeIngest(t, h, func(r *http.Request) {
+		r.Header.Set("Authorization", "Bearer real-token")
+	})
+	if rec.Code != http.StatusOK {
+		t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String())
+	}
+	var body authorizeIngestResponse
+	if err := json.NewDecoder(rec.Body).Decode(&body); err != nil {
+		t.Fatalf("decoding response: %v", err)
+	}
+	if body.TenantID != "acme" {
+		t.Fatalf("TenantID = %q, want acme", body.TenantID)
+	}
+}
+
+func TestAuthorizeIngestNoCredentialsIsUnauthorized(t *testing.T) {
+	h, _ := testHandler(t)
+	rec := doAuthorizeIngest(t, h, nil)
+	if rec.Code != http.StatusUnauthorized {
+		t.Fatalf("status = %d, want 401", rec.Code)
+	}
+}
+
+func TestAuthorizeIngestUnknownTokenIsUnauthorized(t *testing.T) {
+	h, _ := testHandler(t)
+	rec := doAuthorizeIngest(t, h, func(r *http.Request) {
+		r.Header.Set("Authorization", "Bearer not-a-real-token")
+	})
+	if rec.Code != http.StatusUnauthorized {
+		t.Fatalf("status = %d, want 401", rec.Code)
+	}
+}
+
+// TestAuthorizeIngestRejectsSessionToken is the regression test for the
+// two /internal/authorize* endpoints validating genuinely different
+// credential types: a real session.Manager-signed token (a service
+// token or human session) must not work as an ingest credential, since
+// it was never checked against rbacstore.ValidateIngestCredential --
+// this endpoint doesn't call session.Manager.Validate at all.
+func TestAuthorizeIngestRejectsSessionToken(t *testing.T) {
+	h, m := testHandler(t)
+	sessionToken, err := m.IssueServiceToken("alerting")
+	if err != nil {
+		t.Fatalf("IssueServiceToken: %v", err)
+	}
+	rec := doAuthorizeIngest(t, h, func(r *http.Request) {
+		r.Header.Set("Authorization", "Bearer "+sessionToken)
+	})
+	if rec.Code != http.StatusUnauthorized {
+		t.Fatalf("status = %d, want 401 (a session token must not validate as an ingest credential)", rec.Code)
+	}
+}

enterprise/internal/rbacstore/ingest_credentials.go

@@ -0,0 +1,110 @@
+package rbacstore
+
+import (
+	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/jackc/pgx/v5"
+)
+
+// IngestCredential is one ingest_credentials row -- see
+// metadata/migrations/0034_create_ingest_credentials.sql's doc comment
+// for why only a hash is stored. Consumed by
+// ingest/internal/grpcserver.TenantResolver (an HTTP call to
+// enterprise-auth's POST /internal/authorize-ingest, which calls
+// ValidateIngestCredential below) so an agent's records can be
+// attributed to a tenant at the point they enter the system.
+type IngestCredential struct {
+	ID        string
+	TenantID  string
+	CreatedAt time.Time
+}
+
+func hashIngestToken(token string) string {
+	sum := sha256.Sum256([]byte(token))
+	return hex.EncodeToString(sum[:])
+}
+
+// CreateIngestCredential generates a new bearer token for tenantID and
+// returns the plaintext exactly once -- only its hash is ever persisted
+// (see this file's package doc comment). There is no way to retrieve a
+// lost token again; the only recovery is issuing a new one
+// (RevokeIngestCredential + CreateIngestCredential), the same "can't
+// recover, can only reissue" UX every real API-key system uses.
+func (s *Store) CreateIngestCredential(ctx context.Context, tenantID string) (token string, err error) {
+	raw := make([]byte, 32)
+	if _, err := rand.Read(raw); err != nil {
+		return "", fmt.Errorf("rbacstore: generating ingest credential: %w", err)
+	}
+	token = base64.RawURLEncoding.EncodeToString(raw)
+
+	_, err = s.pool.Exec(ctx, `
+		INSERT INTO ingest_credentials (id, tenant_id, token_hash)
+		VALUES ($1, $2, $3)`,
+		uuid.NewString(), tenantID, hashIngestToken(token))
+	if err != nil {
+		return "", fmt.Errorf("rbacstore: creating ingest credential: %w", err)
+	}
+	return token, nil
+}
+
+// ValidateIngestCredential hashes the presented token and looks up which
+// tenant it belongs to via an indexed exact-match on the UNIQUE
+// token_hash column -- the only production call site is
+// enterprise-auth's POST /internal/authorize-ingest handler, per an
+// agent's PushBatch request.
+func (s *Store) ValidateIngestCredential(ctx context.Context, token string) (tenantID string, err error) {
+	row := s.pool.QueryRow(ctx, `SELECT tenant_id FROM ingest_credentials WHERE token_hash = $1`, hashIngestToken(token))
+	if err := row.Scan(&tenantID); err != nil {
+		if errors.Is(err, pgx.ErrNoRows) {
+			return "", ErrNotFound
+		}
+		return "", fmt.Errorf("rbacstore: validating ingest credential: %w", err)
+	}
+	return tenantID, nil
+}
+
+// RevokeIngestCredential deletes a credential by ID (not by token --
+// the plaintext is never stored, so revocation has to name the row some
+// other way; ListIngestCredentialsForTenant is what an operator uses to
+// find the ID).
+func (s *Store) RevokeIngestCredential(ctx context.Context, id string) error {
+	tag, err := s.pool.Exec(ctx, `DELETE FROM ingest_credentials WHERE id = $1`, id)
+	if err != nil {
+		return fmt.Errorf("rbacstore: revoking ingest credential: %w", err)
+	}
+	if tag.RowsAffected() == 0 {
+		return ErrNotFound
+	}
+	return nil
+}
+
+// ListIngestCredentialsForTenant never returns the plaintext token (it
+// isn't stored) -- just enough (ID, creation time) for an operator to
+// decide which one to revoke.
+func (s *Store) ListIngestCredentialsForTenant(ctx context.Context, tenantID string) ([]IngestCredential, error) {
+	rows, err := s.pool.Query(ctx, `
+		SELECT id, tenant_id, created_at FROM ingest_credentials
+		WHERE tenant_id = $1 ORDER BY created_at`, tenantID)
+	if err != nil {
+		return nil, fmt.Errorf("rbacstore: listing ingest credentials: %w", err)
+	}
+	defer rows.Close()
+
+	var out []IngestCredential
+	for rows.Next() {
+		var c IngestCredential
+		if err := rows.Scan(&c.ID, &c.TenantID, &c.CreatedAt); err != nil {
+			return nil, fmt.Errorf("rbacstore: scanning ingest credential: %w", err)
+		}
+		out = append(out, c)
+	}
+	return out, rows.Err()
+}

enterprise/internal/rbacstore/rbacstore_test.go

@@ -771,3 +771,122 @@ func TestListMembershipsForTenant(t *testing.T) {
 		t.Fatalf("unexpected editor entry: %+v", got)
 	}
 }
+
+func TestCreateAndValidateIngestCredential(t *testing.T) {
+	s := testStore(t)
+	ctx := context.Background()
+	tenantID := "test-tenant-" + uniqueSuffix()
+	if _, err := s.CreateTenant(ctx, tenantID, "Test Tenant"); err != nil {
+		t.Fatalf("CreateTenant: %v", err)
+	}
+
+	token, err := s.CreateIngestCredential(ctx, tenantID)
+	if err != nil {
+		t.Fatalf("CreateIngestCredential: %v", err)
+	}
+	if token == "" {
+		t.Fatal("expected a non-empty token")
+	}
+
+	got, err := s.ValidateIngestCredential(ctx, token)
+	if err != nil {
+		t.Fatalf("ValidateIngestCredential: %v", err)
+	}
+	if got != tenantID {
+		t.Fatalf("ValidateIngestCredential tenant = %q, want %q", got, tenantID)
+	}
+}
+
+func TestValidateIngestCredentialRejectsUnknownToken(t *testing.T) {
+	s := testStore(t)
+	if _, err := s.ValidateIngestCredential(context.Background(), "not-a-real-token"); err != ErrNotFound {
+		t.Fatalf("ValidateIngestCredential error = %v, want ErrNotFound", err)
+	}
+}
+
+// TestIngestCredentialTokenNeverStoredAsPlaintext is the regression test
+// for this table's whole reason for hashing: the raw token string must
+// not appear anywhere in the persisted row (only its hash), so a
+// database leak doesn't hand out usable credentials.
+func TestIngestCredentialTokenNeverStoredAsPlaintext(t *testing.T) {
+	s := testStore(t)
+	ctx := context.Background()
+	tenantID := "test-tenant-" + uniqueSuffix()
+	if _, err := s.CreateTenant(ctx, tenantID, "Test Tenant"); err != nil {
+		t.Fatalf("CreateTenant: %v", err)
+	}
+	token, err := s.CreateIngestCredential(ctx, tenantID)
+	if err != nil {
+		t.Fatalf("CreateIngestCredential: %v", err)
+	}
+
+	var stored string
+	row := s.pool.QueryRow(ctx, `SELECT token_hash FROM ingest_credentials WHERE tenant_id = $1`, tenantID)
+	if err := row.Scan(&stored); err != nil {
+		t.Fatalf("reading stored token_hash: %v", err)
+	}
+	if stored == token {
+		t.Fatal("the plaintext token must never be stored directly in token_hash")
+	}
+	if stored != hashIngestToken(token) {
+		t.Fatalf("stored hash = %q, want sha256(token) = %q", stored, hashIngestToken(token))
+	}
+}
+
+func TestRevokeIngestCredential(t *testing.T) {
+	s := testStore(t)
+	ctx := context.Background()
+	tenantID := "test-tenant-" + uniqueSuffix()
+	if _, err := s.CreateTenant(ctx, tenantID, "Test Tenant"); err != nil {
+		t.Fatalf("CreateTenant: %v", err)
+	}
+	token, err := s.CreateIngestCredential(ctx, tenantID)
+	if err != nil {
+		t.Fatalf("CreateIngestCredential: %v", err)
+	}
+	creds, err := s.ListIngestCredentialsForTenant(ctx, tenantID)
+	if err != nil || len(creds) != 1 {
+		t.Fatalf("ListIngestCredentialsForTenant = (%+v, %v), want exactly one", creds, err)
+	}
+
+	if err := s.RevokeIngestCredential(ctx, creds[0].ID); err != nil {
+		t.Fatalf("RevokeIngestCredential: %v", err)
+	}
+	if _, err := s.ValidateIngestCredential(ctx, token); err != ErrNotFound {
+		t.Fatalf("ValidateIngestCredential after revoke = %v, want ErrNotFound", err)
+	}
+}
+
+func TestRevokeIngestCredentialNotFound(t *testing.T) {
+	s := testStore(t)
+	if err := s.RevokeIngestCredential(context.Background(), uuid.NewString()); err != ErrNotFound {
+		t.Fatalf("RevokeIngestCredential error = %v, want ErrNotFound", err)
+	}
+}
+
+func TestListIngestCredentialsForTenantExcludesOtherTenants(t *testing.T) {
+	s := testStore(t)
+	ctx := context.Background()
+	tenantID := "test-tenant-" + uniqueSuffix()
+	otherTenantID := "test-tenant-" + uniqueSuffix()
+	if _, err := s.CreateTenant(ctx, tenantID, "Test Tenant"); err != nil {
+		t.Fatalf("CreateTenant: %v", err)
+	}
+	if _, err := s.CreateTenant(ctx, otherTenantID, "Other Tenant"); err != nil {
+		t.Fatalf("CreateTenant other: %v", err)
+	}
+	if _, err := s.CreateIngestCredential(ctx, tenantID); err != nil {
+		t.Fatalf("CreateIngestCredential: %v", err)
+	}
+	if _, err := s.CreateIngestCredential(ctx, otherTenantID); err != nil {
+		t.Fatalf("CreateIngestCredential other: %v", err)
+	}
+
+	creds, err := s.ListIngestCredentialsForTenant(ctx, tenantID)
+	if err != nil {
+		t.Fatalf("ListIngestCredentialsForTenant: %v", err)
+	}
+	if len(creds) != 1 || creds[0].TenantID != tenantID {
+		t.Fatalf("unexpected credentials: %+v", creds)
+	}
+}