Files
cairnobs/api/agents/store.go
T
jcoffey-dev 4f0da1ae5e Add agent inventory, management, and remote config
Extends the heartbeat mechanism with a second gRPC service on the same
mTLS channel (AgentControl.CheckIn, agent-initiated on the existing
heartbeat ticker -- still push-only, no inbound port on any agent) so
an agent reports its running config and can pick up an operator-set
override. A new web UI section (/agents) lists every agent that's
checked in, shows its reported config, and lets an operator edit a
narrow, deliberately-scoped subset remotely: batch/heartbeat tuning,
and (journald sources only) the unit filter.

TLS material and the ingest endpoint are never reportable or remotely
editable, by proto shape rather than a validation rule -- a bad or
malicious edit there could permanently strand an agent or redirect
where its logs go, unlike every other editable field, which only
degrades behavior.

An override lives only in the agent's memory (agent.toml is never
rewritten) and re-syncs on the agent's own schedule; changing the
journald filter aborts and respawns the source task since there's no
other way to change what's being tailed. Building the hot-reload path
surfaced a real, independent, pre-existing bug: shutdown was using
poll_timeout(), which only drains once flush_interval has elapsed,
silently dropping anything buffered more recently on every graceful
shutdown that landed between flushes -- fixed with a new unconditional
Batcher::flush_all(), now used at both shutdown and hot-reload.

Verified live end-to-end against a real stack: an edited heartbeat
interval changed a running agent's actual send cadence within one
check-in cycle (confirmed by the real timestamps landing in
ClickHouse), and an edited journald filter triggered a real source
restart, both reflected back in the next reported-config snapshot.

See /docs/agent-management-design.md.
2026-08-16 18:08:51 -07:00

210 lines
7.1 KiB
Go

// Package agents is the web-facing half of agent inventory/remote
// config (see /docs/agent-management-design.md) -- reads/writes the
// same `agents` table ingest's internal/agentregistry writes on every
// CheckIn RPC, the same shared-schema-different-services shape
// alerting and api already use for dashboards/alert_rules.
package agents
import (
"context"
"encoding/json"
"errors"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
var ErrNotFound = errors.New("not found")
// ConfigOverride is the remotely-editable subset of an agent's config --
// a plain-Go mirror of ingest/internal/agentregistry's overrideFields
// and agent_control.proto's DesiredOverride. Deliberately duplicated
// rather than imported across the module boundary, same convention as
// every other cross-module shared shape in this codebase (see
// grpcserver.TenantIDHeaderKey, enterprise/internal/apiconfig.AIConfig).
// Keep the three in sync by hand.
type ConfigOverride struct {
BatchMaxSize *int64 `json:"batch_max_size,omitempty"`
BatchFlushIntervalMS *int64 `json:"batch_flush_interval_ms,omitempty"`
HeartbeatEnabled *bool `json:"heartbeat_enabled,omitempty"`
HeartbeatIntervalMS *int64 `json:"heartbeat_interval_ms,omitempty"`
JournaldUnit *string `json:"journald_unit,omitempty"`
}
type Agent struct {
ID string `json:"id"`
TenantID string `json:"tenant_id"`
Host string `json:"host"`
Service string `json:"service"`
AgentVersion string `json:"agent_version"`
SourceKind string `json:"source_kind"`
SourceDetail string `json:"source_detail"`
BatchMaxSize int64 `json:"batch_max_size"`
BatchFlushIntervalMS int64 `json:"batch_flush_interval_ms"`
HeartbeatEnabled bool `json:"heartbeat_enabled"`
HeartbeatIntervalMS int64 `json:"heartbeat_interval_ms"`
FirstSeenAt time.Time `json:"first_seen_at"`
LastSeenAt time.Time `json:"last_seen_at"`
DesiredOverride *ConfigOverride `json:"desired_override,omitempty"`
DesiredOverrideVersion string `json:"desired_override_version,omitempty"`
AppliedOverrideVersion string `json:"applied_override_version"`
// Pending is computed, not stored: an override exists
// (DesiredOverrideVersion != "") that the agent hasn't reported
// applying yet (AppliedOverrideVersion doesn't match). This is what
// the web UI's "pending"/"applied" indicator (task selected:
// "+ Remote config editing") reads directly, rather than
// recomputing the same string comparison itself.
Pending bool `json:"pending"`
UpdatedBy string `json:"updated_by,omitempty"`
}
type Store struct {
pool *pgxpool.Pool
}
func NewStore(pool *pgxpool.Pool) *Store {
return &Store{pool: pool}
}
const selectColumns = `
id, tenant_id, host, service,
reported_agent_version, reported_source_kind, reported_source_detail,
reported_batch_max_size, reported_batch_flush_ms,
reported_heartbeat_on, reported_heartbeat_ms,
first_seen_at, last_seen_at,
desired_override, desired_override_version, applied_override_version, updated_by`
func (s *Store) List(ctx context.Context, tenantID string) ([]Agent, error) {
rows, err := s.pool.Query(ctx, `
SELECT `+selectColumns+`
FROM agents WHERE tenant_id = $1 ORDER BY host`, tenantID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Agent
for rows.Next() {
a, err := scanAgent(rows)
if err != nil {
return nil, err
}
out = append(out, a)
}
return out, rows.Err()
}
func (s *Store) Get(ctx context.Context, tenantID, host string) (*Agent, error) {
rows, err := s.pool.Query(ctx, `
SELECT `+selectColumns+`
FROM agents WHERE tenant_id = $1 AND host = $2`, tenantID, host)
if err != nil {
return nil, err
}
defer rows.Close()
if !rows.Next() {
if err := rows.Err(); err != nil {
return nil, err
}
return nil, ErrNotFound
}
a, err := scanAgent(rows)
if err != nil {
return nil, err
}
return &a, nil
}
// SetOverride writes a new desired override for host, generating a
// fresh version stamp -- overwrites any previous override wholesale
// (this is "set the desired config," not "patch a few fields into
// whatever was there," so a caller building a partial edit must have
// already merged it against the current value, same as any other PUT
// endpoint in this codebase). Returns ErrNotFound if the agent has
// never checked in (nothing to target an override at yet -- an
// override for a host ingest has never seen would be silently
// unreachable).
func (s *Store) SetOverride(ctx context.Context, tenantID, host string, override ConfigOverride, updatedBy string) (*Agent, error) {
version := newVersion()
data, err := json.Marshal(override)
if err != nil {
return nil, err
}
tag, err := s.pool.Exec(ctx, `
UPDATE agents SET desired_override = $1, desired_override_version = $2, updated_by = $3
WHERE tenant_id = $4 AND host = $5`,
data, version, updatedBy, tenantID, host)
if err != nil {
return nil, err
}
if tag.RowsAffected() == 0 {
return nil, ErrNotFound
}
return s.Get(ctx, tenantID, host)
}
// ClearOverride reverts an agent to running its local agent.toml
// untouched -- the next CheckIn gets has_override=false.
func (s *Store) ClearOverride(ctx context.Context, tenantID, host string) error {
tag, err := s.pool.Exec(ctx, `
UPDATE agents SET desired_override = NULL, desired_override_version = NULL, updated_by = NULL
WHERE tenant_id = $1 AND host = $2`, tenantID, host)
if err != nil {
return err
}
if tag.RowsAffected() == 0 {
return ErrNotFound
}
return nil
}
type rowScanner interface {
Scan(dest ...any) error
}
func scanAgent(row rowScanner) (Agent, error) {
var a Agent
var desiredOverride []byte
var desiredVersion, updatedBy *string
if err := row.Scan(
&a.ID, &a.TenantID, &a.Host, &a.Service,
&a.AgentVersion, &a.SourceKind, &a.SourceDetail,
&a.BatchMaxSize, &a.BatchFlushIntervalMS,
&a.HeartbeatEnabled, &a.HeartbeatIntervalMS,
&a.FirstSeenAt, &a.LastSeenAt,
&desiredOverride, &desiredVersion, &a.AppliedOverrideVersion, &updatedBy,
); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return Agent{}, ErrNotFound
}
return Agent{}, err
}
if updatedBy != nil {
a.UpdatedBy = *updatedBy
}
if desiredVersion != nil {
a.DesiredOverrideVersion = *desiredVersion
a.Pending = *desiredVersion != a.AppliedOverrideVersion
if len(desiredOverride) > 0 {
var override ConfigOverride
if err := json.Unmarshal(desiredOverride, &override); err != nil {
return Agent{}, err
}
a.DesiredOverride = &override
}
}
return a, nil
}
// newVersion is an opaque, monotonically-informative-enough stamp for
// DesiredOverride.version -- a timestamp, not a counter, since Store
// has no prior version to increment from without an extra read. Never
// interpreted as a real time value by the agent (see
// agent_control.proto's DesiredOverride.version comment) -- just needs
// to change on every edit.
func newVersion() string {
return time.Now().UTC().Format(time.RFC3339Nano)
}