Add agent inventory, management, and remote config

Extends the heartbeat mechanism with a second gRPC service on the same
mTLS channel (AgentControl.CheckIn, agent-initiated on the existing
heartbeat ticker -- still push-only, no inbound port on any agent) so
an agent reports its running config and can pick up an operator-set
override. A new web UI section (/agents) lists every agent that's
checked in, shows its reported config, and lets an operator edit a
narrow, deliberately-scoped subset remotely: batch/heartbeat tuning,
and (journald sources only) the unit filter.

TLS material and the ingest endpoint are never reportable or remotely
editable, by proto shape rather than a validation rule -- a bad or
malicious edit there could permanently strand an agent or redirect
where its logs go, unlike every other editable field, which only
degrades behavior.

An override lives only in the agent's memory (agent.toml is never
rewritten) and re-syncs on the agent's own schedule; changing the
journald filter aborts and respawns the source task since there's no
other way to change what's being tailed. Building the hot-reload path
surfaced a real, independent, pre-existing bug: shutdown was using
poll_timeout(), which only drains once flush_interval has elapsed,
silently dropping anything buffered more recently on every graceful
shutdown that landed between flushes -- fixed with a new unconditional
Batcher::flush_all(), now used at both shutdown and hot-reload.

Verified live end-to-end against a real stack: an edited heartbeat
interval changed a running agent's actual send cadence within one
check-in cycle (confirmed by the real timestamps landing in
ClickHouse), and an edited journald filter triggered a real source
restart, both reflected back in the next reported-config snapshot.

See /docs/agent-management-design.md.
This commit is contained in:
2026-08-16 18:08:51 -07:00
parent 4df6931869
commit 4f0da1ae5e
29 changed files with 2618 additions and 53 deletions
@@ -0,0 +1,108 @@
// Package agentregistry is the Postgres-backed implementation of
// grpcserver.AgentRegistry -- ingest's half of agent inventory/remote
// config (see /docs/agent-management-design.md). Writes into the same
// sentry_metadata Postgres api reads/writes from for the web UI's
// inventory and edit-config views (api/agents), the same shared-schema-
// different-services shape alerting and api already use for dashboards/
// alert_rules.
package agentregistry
import (
"context"
"encoding/json"
"fmt"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/sentry/sentry/ingest/internal/grpcserver"
)
// defaultTenantID is used when no TenantResolver is configured (empty
// tenantID from grpcserver) -- the same 'default' tenant row every
// single-tenant deployment's Postgres already has, seeded by
// metadata/migrations/0019_seed_default_tenant.sql.
const defaultTenantID = "default"
// overrideFields is the JSON shape stored in agents.desired_override.
// Deliberately duplicated in api/agents rather than imported -- these
// are two different Go modules, and this codebase's established
// convention (see enterprise/internal/apiconfig.AIConfig,
// grpcserver.TenantIDHeaderKey) is to duplicate a small shared shape
// across a module boundary rather than couple two independently
// deployable services' builds together. Keep the two in sync by hand.
type overrideFields struct {
BatchMaxSize *uint64 `json:"batch_max_size,omitempty"`
BatchFlushIntervalMS *uint64 `json:"batch_flush_interval_ms,omitempty"`
HeartbeatEnabled *bool `json:"heartbeat_enabled,omitempty"`
HeartbeatIntervalMS *uint64 `json:"heartbeat_interval_ms,omitempty"`
JournaldUnit *string `json:"journald_unit,omitempty"`
}
type Registry struct {
pool *pgxpool.Pool
}
func New(pool *pgxpool.Pool) *Registry {
return &Registry{pool: pool}
}
var _ grpcserver.AgentRegistry = (*Registry)(nil)
func (r *Registry) CheckIn(ctx context.Context, tenantID string, info grpcserver.AgentCheckIn) (grpcserver.AgentOverride, error) {
if tenantID == "" {
tenantID = defaultTenantID
}
var (
desiredOverride []byte
desiredVersion *string
)
err := r.pool.QueryRow(ctx, `
INSERT INTO agents (
id, tenant_id, host, service,
reported_agent_version, reported_source_kind, reported_source_detail,
reported_batch_max_size, reported_batch_flush_ms,
reported_heartbeat_on, reported_heartbeat_ms,
first_seen_at, last_seen_at, applied_override_version
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11, now(), now(), $12)
ON CONFLICT (tenant_id, host) DO UPDATE SET
service = EXCLUDED.service,
reported_agent_version = EXCLUDED.reported_agent_version,
reported_source_kind = EXCLUDED.reported_source_kind,
reported_source_detail = EXCLUDED.reported_source_detail,
reported_batch_max_size = EXCLUDED.reported_batch_max_size,
reported_batch_flush_ms = EXCLUDED.reported_batch_flush_ms,
reported_heartbeat_on = EXCLUDED.reported_heartbeat_on,
reported_heartbeat_ms = EXCLUDED.reported_heartbeat_ms,
last_seen_at = now(),
applied_override_version = EXCLUDED.applied_override_version
RETURNING desired_override, desired_override_version`,
uuid.NewString(), tenantID, info.Host, info.Service,
info.AgentVersion, info.SourceKind, info.SourceDetail,
info.BatchMaxSize, info.BatchFlushIntervalMS,
info.HeartbeatEnabled, info.HeartbeatIntervalMS,
info.AppliedOverrideVersion,
).Scan(&desiredOverride, &desiredVersion)
if err != nil {
return grpcserver.AgentOverride{}, fmt.Errorf("agentregistry: upserting check-in: %w", err)
}
if desiredVersion == nil || len(desiredOverride) == 0 {
return grpcserver.AgentOverride{HasOverride: false}, nil
}
var fields overrideFields
if err := json.Unmarshal(desiredOverride, &fields); err != nil {
return grpcserver.AgentOverride{}, fmt.Errorf("agentregistry: parsing stored override: %w", err)
}
return grpcserver.AgentOverride{
HasOverride: true,
BatchMaxSize: fields.BatchMaxSize,
BatchFlushIntervalMS: fields.BatchFlushIntervalMS,
HeartbeatEnabled: fields.HeartbeatEnabled,
HeartbeatIntervalMS: fields.HeartbeatIntervalMS,
JournaldUnit: fields.JournaldUnit,
Version: *desiredVersion,
}, nil
}
+28
View File
@@ -23,6 +23,26 @@ type Config struct {
// as every other optional enterprise integration point in this
// codebase (e.g. api's own ENTERPRISE_AUTH_URL).
EnterpriseAuthURL string
// AgentRegistry enables agent inventory/remote config
// (internal/agentregistry, internal/grpcserver.AgentRegistry) when
// Postgres.Addr is set -- same "off unless configured" shape as
// EnterpriseAuthURL above. Writes into the same sentry_metadata
// database api/web already use, via the same shared "sentry" role
// every other non-audit table in this schema uses (unlike
// audit_log's dedicated restricted role -- agent inventory carries
// no tamper-evidence requirement).
AgentRegistry AgentRegistryConfig
}
type AgentRegistryConfig struct {
Postgres PostgresConfig
}
type PostgresConfig struct {
Addr string
Database string
Username string
Password string
}
type GRPCConfig struct {
@@ -77,6 +97,14 @@ func Load() (Config, error) {
Password: getenv("CLICKHOUSE_PASSWORD", ""),
},
EnterpriseAuthURL: getenv("ENTERPRISE_AUTH_URL", ""),
AgentRegistry: AgentRegistryConfig{
Postgres: PostgresConfig{
Addr: getenv("AGENT_REGISTRY_POSTGRES_ADDR", ""),
Database: getenv("AGENT_REGISTRY_POSTGRES_DATABASE", "sentry_metadata"),
Username: getenv("AGENT_REGISTRY_POSTGRES_USERNAME", "sentry"),
Password: getenv("AGENT_REGISTRY_POSTGRES_PASSWORD", ""),
},
},
}
maxSize, err := strconv.Atoi(getenv("CONSUMER_BATCH_MAX_SIZE", "500"))
+131 -22
View File
@@ -39,6 +39,7 @@ import (
"google.golang.org/protobuf/proto"
"github.com/sentry/sentry/ingest/internal/config"
agentv1 "github.com/sentry/sentry/proto/sentry/agent/v1"
logsv1 "github.com/sentry/sentry/proto/sentry/logs/v1"
)
@@ -52,12 +53,14 @@ const TenantIDHeaderKey = "tenant_id"
type Server struct {
logsv1.UnimplementedLogIngestServer
agentv1.UnimplementedAgentControlServer
logger *slog.Logger
grpcCfg config.GRPCConfig
tlsCfg config.TLSConfig
producer batchProducer
resolver TenantResolver
agents AgentRegistry
}
// batchProducer is the subset of *producer.Producer this package depends
@@ -80,8 +83,52 @@ type TenantResolver interface {
ResolveTenant(ctx context.Context, token string) (tenantID string, err error)
}
func New(logger *slog.Logger, grpcCfg config.GRPCConfig, tlsCfg config.TLSConfig, p batchProducer, resolver TenantResolver) *Server {
return &Server{logger: logger, grpcCfg: grpcCfg, tlsCfg: tlsCfg, producer: p, resolver: resolver}
// AgentRegistry records an agent's CheckIn (for the web UI's inventory
// view) and returns any remote config override an operator has set for
// it. nil is a deliberate no-op, same "off unless configured" shape as
// TenantResolver: CheckIn always succeeds and reports "no override" --
// a deployment that hasn't configured AGENT_REGISTRY_POSTGRES_ADDR
// simply doesn't get agent inventory/management, exactly like one
// without ENTERPRISE_AUTH_URL doesn't get tenant-tagged records.
type AgentRegistry interface {
CheckIn(ctx context.Context, tenantID string, info AgentCheckIn) (AgentOverride, error)
}
// AgentCheckIn is what an agent reports about itself on each CheckIn --
// a plain-Go mirror of agentv1.ReportedConfig plus the identity/
// tenant fields, kept separate from the proto type so AgentRegistry
// implementations (ingest/internal/agentregistry) don't need to import
// this package's gRPC-facing types just to satisfy the interface.
type AgentCheckIn struct {
Host string
Service string
AgentVersion string
SourceKind string
SourceDetail string
BatchMaxSize uint64
BatchFlushIntervalMS uint64
HeartbeatEnabled bool
HeartbeatIntervalMS uint64
AppliedOverrideVersion string
}
// AgentOverride is the remotely-editable subset of an agent's config, as
// currently stored for it -- a plain-Go mirror of agentv1.DesiredOverride.
// Every pointer field is nil when that field has no override set.
// HasOverride false means no override has ever been set at all (Version
// is meaningless in that case).
type AgentOverride struct {
HasOverride bool
BatchMaxSize *uint64
BatchFlushIntervalMS *uint64
HeartbeatEnabled *bool
HeartbeatIntervalMS *uint64
JournaldUnit *string
Version string
}
func New(logger *slog.Logger, grpcCfg config.GRPCConfig, tlsCfg config.TLSConfig, p batchProducer, resolver TenantResolver, agents AgentRegistry) *Server {
return &Server{logger: logger, grpcCfg: grpcCfg, tlsCfg: tlsCfg, producer: p, resolver: resolver, agents: agents}
}
// Run blocks serving gRPC until ctx is canceled, then gracefully stops.
@@ -98,6 +145,7 @@ func (s *Server) Run(ctx context.Context) error {
grpcSrv := grpc.NewServer(grpc.Creds(credentials.NewTLS(tlsConf)))
logsv1.RegisterLogIngestServer(grpcSrv, s)
agentv1.RegisterAgentControlServer(grpcSrv, s)
s.logger.Info("gRPC server listening", "addr", s.grpcCfg.ListenAddr)
@@ -118,26 +166,10 @@ func (s *Server) PushBatch(ctx context.Context, req *logsv1.PushBatchRequest) (*
return &logsv1.PushBatchResponse{Accepted: 0}, nil
}
// tenantID stays empty (no header attached below) unless a resolver
// is actually configured -- single-tenant deployments never present
// a bearer credential and never need to. Once a resolver IS
// configured, a missing/invalid credential fails the whole batch
// closed rather than falling back to "no tenant" -- exactly the
// same fail-closed shape enterprise/internal/chrunner.Registry.RunSQL
// uses on the read side, applied here at the point data enters the
// system.
var tenantID string
if s.resolver != nil {
token, ok := bearerTokenFromContext(ctx)
if !ok {
return nil, status.Error(codes.Unauthenticated, "missing bearer credential")
}
resolved, err := s.resolver.ResolveTenant(ctx, token)
if err != nil {
s.logger.Error("resolving ingest tenant", "batch_id", req.GetBatchId(), "error", err)
return nil, status.Error(codes.Unauthenticated, "invalid ingest credential")
}
tenantID = resolved
tenantID, err := s.resolveTenant(ctx)
if err != nil {
s.logger.Error("resolving ingest tenant", "batch_id", req.GetBatchId(), "error", err)
return nil, err
}
msgs := make([]kafka.Message, 0, len(req.GetRecords()))
@@ -174,6 +206,83 @@ func (s *Server) PushBatch(ctx context.Context, req *logsv1.PushBatchRequest) (*
return &logsv1.PushBatchResponse{Accepted: uint32(len(req.GetRecords()))}, nil
}
// resolveTenant is PushBatch's and CheckIn's shared tenant-resolution
// step, extracted so CheckIn gets the identical fail-closed behavior
// without duplicating it: empty tenantID (no resolver configured, the
// single-tenant default) is not an error, but a configured resolver
// that gets no/an invalid credential is -- exactly the same posture
// enterprise/internal/chrunner.Registry.RunSQL uses on the read side,
// applied here at the point data (or a check-in) enters the system.
func (s *Server) resolveTenant(ctx context.Context) (string, error) {
if s.resolver == nil {
return "", nil
}
token, ok := bearerTokenFromContext(ctx)
if !ok {
return "", status.Error(codes.Unauthenticated, "missing bearer credential")
}
resolved, err := s.resolver.ResolveTenant(ctx, token)
if err != nil {
return "", status.Error(codes.Unauthenticated, "invalid ingest credential")
}
return resolved, nil
}
// CheckIn is AgentControl's one RPC (see agent_control.proto) -- agent-
// initiated, on its own heartbeat ticker. A nil AgentRegistry (no
// AGENT_REGISTRY_POSTGRES_ADDR configured) makes this a pure no-op that
// always reports "no override," so agents calling in against a
// deployment that hasn't opted into this feature see no behavior
// change at all.
func (s *Server) CheckIn(ctx context.Context, req *agentv1.CheckInRequest) (*agentv1.CheckInResponse, error) {
if req.GetHost() == "" {
return nil, status.Error(codes.InvalidArgument, "host must not be empty")
}
tenantID, err := s.resolveTenant(ctx)
if err != nil {
s.logger.Error("resolving ingest tenant for check-in", "host", req.GetHost(), "error", err)
return nil, err
}
if s.agents == nil {
return &agentv1.CheckInResponse{HasOverride: false}, nil
}
cfg := req.GetCurrentConfig()
override, err := s.agents.CheckIn(ctx, tenantID, AgentCheckIn{
Host: req.GetHost(),
Service: req.GetService(),
AgentVersion: cfg.GetAgentVersion(),
SourceKind: cfg.GetSourceKind(),
SourceDetail: cfg.GetSourceDetail(),
BatchMaxSize: cfg.GetBatchMaxSize(),
BatchFlushIntervalMS: cfg.GetBatchFlushIntervalMs(),
HeartbeatEnabled: cfg.GetHeartbeatEnabled(),
HeartbeatIntervalMS: cfg.GetHeartbeatIntervalMs(),
AppliedOverrideVersion: req.GetAppliedOverrideVersion(),
})
if err != nil {
s.logger.Error("recording agent check-in", "host", req.GetHost(), "error", err)
return nil, status.Errorf(codes.Internal, "recording check-in: %v", err)
}
if !override.HasOverride {
return &agentv1.CheckInResponse{HasOverride: false}, nil
}
return &agentv1.CheckInResponse{
HasOverride: true,
Override: &agentv1.DesiredOverride{
BatchMaxSize: override.BatchMaxSize,
BatchFlushIntervalMs: override.BatchFlushIntervalMS,
HeartbeatEnabled: override.HeartbeatEnabled,
HeartbeatIntervalMs: override.HeartbeatIntervalMS,
JournaldUnit: override.JournaldUnit,
Version: override.Version,
},
}, nil
}
// bearerTokenFromContext reads the same "authorization: Bearer <token>"
// gRPC metadata shape HTTP's Authorization header uses -- an agent sets
// this once per PushBatch call (see the agent's grpc.rs), not per
+123 -2
View File
@@ -15,6 +15,7 @@ import (
"google.golang.org/protobuf/proto"
"github.com/sentry/sentry/ingest/internal/config"
agentv1 "github.com/sentry/sentry/proto/sentry/agent/v1"
logsv1 "github.com/sentry/sentry/proto/sentry/logs/v1"
)
@@ -50,12 +51,40 @@ func (f *fakeResolver) ResolveTenant(_ context.Context, token string) (string, e
return tenantID, nil
}
// fakeAgentRegistry is an in-memory stand-in for
// ingest/internal/agentregistry.Registry, keyed by "tenantID/host" so
// tests can assert cross-tenant isolation the same way the real
// UNIQUE (tenant_id, host) constraint provides it.
type fakeAgentRegistry struct {
mu sync.Mutex
checkIns []AgentCheckIn
overrides map[string]AgentOverride
err error
}
func (f *fakeAgentRegistry) CheckIn(_ context.Context, tenantID string, info AgentCheckIn) (AgentOverride, error) {
f.mu.Lock()
defer f.mu.Unlock()
if f.err != nil {
return AgentOverride{}, f.err
}
f.checkIns = append(f.checkIns, info)
if f.overrides == nil {
return AgentOverride{HasOverride: false}, nil
}
return f.overrides[tenantID+"/"+info.Host], nil
}
func newTestServer(p batchProducer) *Server {
return New(slog.New(slog.NewTextHandler(io.Discard, nil)), config.GRPCConfig{}, config.TLSConfig{}, p, nil)
return New(slog.New(slog.NewTextHandler(io.Discard, nil)), config.GRPCConfig{}, config.TLSConfig{}, p, nil, nil)
}
func newTestServerWithResolver(p batchProducer, resolver TenantResolver) *Server {
return New(slog.New(slog.NewTextHandler(io.Discard, nil)), config.GRPCConfig{}, config.TLSConfig{}, p, resolver)
return New(slog.New(slog.NewTextHandler(io.Discard, nil)), config.GRPCConfig{}, config.TLSConfig{}, p, resolver, nil)
}
func newTestServerWithAgents(agents AgentRegistry) *Server {
return New(slog.New(slog.NewTextHandler(io.Discard, nil)), config.GRPCConfig{}, config.TLSConfig{}, &fakeProducer{}, nil, agents)
}
// contextWithBearerToken builds an incoming gRPC context carrying an
@@ -253,3 +282,95 @@ func TestPushBatchWithResolverRejectsInvalidToken(t *testing.T) {
t.Fatal("a batch with an invalid token must never reach the producer once a resolver is configured")
}
}
func TestCheckInNilRegistryIsANoOp(t *testing.T) {
s := newTestServer(&fakeProducer{})
resp, err := s.CheckIn(context.Background(), &agentv1.CheckInRequest{Host: "h1", CurrentConfig: &agentv1.ReportedConfig{}})
if err != nil {
t.Fatalf("CheckIn() error = %v", err)
}
if resp.GetHasOverride() {
t.Fatal("expected has_override=false with no AgentRegistry configured")
}
}
func TestCheckInRejectsEmptyHost(t *testing.T) {
s := newTestServer(&fakeProducer{})
_, err := s.CheckIn(context.Background(), &agentv1.CheckInRequest{CurrentConfig: &agentv1.ReportedConfig{}})
if status.Code(err) != codes.InvalidArgument {
t.Fatalf("CheckIn() error = %v, want InvalidArgument", err)
}
}
func TestCheckInRecordsReportedConfig(t *testing.T) {
reg := &fakeAgentRegistry{}
s := newTestServerWithAgents(reg)
_, err := s.CheckIn(context.Background(), &agentv1.CheckInRequest{
Host: "web-01",
Service: "web",
CurrentConfig: &agentv1.ReportedConfig{
AgentVersion: "0.1.0",
SourceKind: "journald",
BatchMaxSize: 500,
BatchFlushIntervalMs: 2000,
HeartbeatEnabled: true,
HeartbeatIntervalMs: 60000,
},
AppliedOverrideVersion: "v3",
})
if err != nil {
t.Fatalf("CheckIn() error = %v", err)
}
reg.mu.Lock()
defer reg.mu.Unlock()
if len(reg.checkIns) != 1 {
t.Fatalf("expected 1 recorded check-in, got %d", len(reg.checkIns))
}
got := reg.checkIns[0]
if got.Host != "web-01" || got.AgentVersion != "0.1.0" || got.BatchMaxSize != 500 || got.AppliedOverrideVersion != "v3" {
t.Fatalf("unexpected recorded check-in: %+v", got)
}
}
// TestCheckInReturnsOverrideWhenSet is the regression test for the
// actual point of this RPC: an operator-set override for this specific
// host comes back in the response, correctly shaped.
func TestCheckInReturnsOverrideWhenSet(t *testing.T) {
// Keyed by "" (empty tenantID) rather than "default" -- the
// empty-to-"default" substitution is agentregistry.Registry's own
// Postgres-specific behavior (matching the seeded default tenant
// row), not something grpcserver itself does; this fake exercises
// grpcserver.CheckIn in isolation, so it sees the tenantID exactly
// as resolveTenant produced it (empty, since no resolver is
// configured for this test).
interval := uint64(30000)
reg := &fakeAgentRegistry{overrides: map[string]AgentOverride{
"/web-01": {HasOverride: true, HeartbeatIntervalMS: &interval, Version: "v2"},
}}
s := newTestServerWithAgents(reg)
resp, err := s.CheckIn(context.Background(), &agentv1.CheckInRequest{Host: "web-01", CurrentConfig: &agentv1.ReportedConfig{}})
if err != nil {
t.Fatalf("CheckIn() error = %v", err)
}
if !resp.GetHasOverride() {
t.Fatal("expected has_override=true")
}
if resp.GetOverride().GetHeartbeatIntervalMs() != 30000 || resp.GetOverride().GetVersion() != "v2" {
t.Fatalf("unexpected override: %+v", resp.GetOverride())
}
}
func TestCheckInWithResolverRejectsMissingToken(t *testing.T) {
resolver := &fakeResolver{tenantByToken: map[string]string{"real-token": "acme"}}
s := New(slog.New(slog.NewTextHandler(io.Discard, nil)), config.GRPCConfig{}, config.TLSConfig{}, &fakeProducer{}, resolver, &fakeAgentRegistry{})
_, err := s.CheckIn(context.Background(), &agentv1.CheckInRequest{Host: "web-01", CurrentConfig: &agentv1.ReportedConfig{}})
if status.Code(err) != codes.Unauthenticated {
t.Fatalf("CheckIn() error = %v, want Unauthenticated", err)
}
}