jcoffey-dev
bcb9a01cd6
The demo had 75k generic records across eight host-0N/service pairs, one dashboard, one alert rule, and -- because nothing ever called AgentControl.CheckIn -- a completely empty Agents page. /hack/demo-simulator replaces the generic data with a fictional but coherent fleet: 14 hosts running nginx, an API tier, workers, Postgres, Redis, mail, Linux journals and Windows event logs, whose messages and attributes look like what those services actually write. It backfills a week (~370k records, ~20s) and then keeps running. Running continuously is the point, not an implementation detail. Three things the demo has to show are only true if data keeps arriving: the Agents page marks a host stale once check-ins stop, alert rules evaluate over trailing windows and would freeze in one state against a static dataset, and any "last 15 minutes" view is empty on data that stopped growing overnight. It also emits metrics/heartbeats and answers CheckIn faithfully enough that the remote-config editor's pending -> applied transition works end to end. Seeded incidents give the data something to find: an api-02 outage with matching slow queries on db-01, 5xx at the edge and cascading job failures; an SSH probe burst; a spam wave; a disk filling up; and one decommissioned host left deliberately stale. /hack/demo-seed holds the rest of the deployment -- the nightly reset, eight dashboards (64 panels, every viz type but line), eleven alert rules across three notification targets, and the systemd unit. Rule thresholds are calibrated against what the simulator actually produces: the first pass had four rules whose thresholds the traffic could never reach and one that fired during normal operation. No line charts: dashboard panels reject the raw-SQL escape hatch, and the pipe language has no time-bucketing, so a real time axis isn't expressible today. Noted in demo-seed/README.md rather than papered over.
86 lines
3.2 KiB
Go
86 lines
3.2 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"hash/fnv"
|
|
"math"
|
|
"math/rand"
|
|
"strconv"
|
|
"time"
|
|
|
|
logsv1 "github.com/cairnobs/cairnobs/proto/sentry/logs/v1"
|
|
)
|
|
|
|
// Metrics records carry the exact attribute contract the Hosts page
|
|
// reads (see web/src/lib/api.ts's getHostMetrics/listMetricsHosts): a
|
|
// `cairnobs.metrics=true` tag plus utilization and static-context
|
|
// fields, shipped as an ordinary tagged LogRecord because the query
|
|
// language maps any non-standard field name to attributes['field'] with
|
|
// automatic numeric casting -- no separate metrics pipeline exists, by
|
|
// design. Heartbeat records use the same trick with
|
|
// `cairnobs.heartbeat=true`, which is what the absence-style alert rules
|
|
// watch for.
|
|
|
|
// phaseOf gives each host its own deterministic offset into the wander
|
|
// functions below, so two hosts with the same baseline don't move in
|
|
// lockstep across the fleet's charts.
|
|
func phaseOf(name string) float64 {
|
|
h := fnv.New32a()
|
|
_, _ = h.Write([]byte(name))
|
|
return float64(h.Sum32()%1000) / 1000 * 2 * math.Pi
|
|
}
|
|
|
|
func clamp(v, lo, hi float64) float64 {
|
|
return math.Max(lo, math.Min(hi, v))
|
|
}
|
|
|
|
// metricsRecord samples host h at time t. backfillStart anchors the
|
|
// disk-growth trend so a host that's "filling up" is at its lowest at
|
|
// the oldest end of the window and its highest right now, in whichever
|
|
// order the samples happen to be generated.
|
|
func metricsRecord(h *host, t, backfillStart time.Time, r *rand.Rand) *logsv1.LogRecord {
|
|
phase := phaseOf(h.name)
|
|
mins := float64(t.Unix()) / 60
|
|
|
|
// Two sine components of different periods plus noise: a slow
|
|
// business-hours swell and a faster one, so a CPU chart looks like a
|
|
// machine doing work rather than a random walk.
|
|
cpu := h.cpuBase * (1 +
|
|
0.45*math.Sin(mins/97+phase) +
|
|
0.2*math.Sin(mins/13+phase*2))
|
|
cpu = clamp(cpu*diurnal(t)+r.NormFloat64()*2.5, 0.4, 99)
|
|
|
|
memFrac := clamp(h.memFrac*(1+0.08*math.Sin(mins/211+phase))+r.NormFloat64()*0.01, 0.03, 0.97)
|
|
|
|
days := t.Sub(backfillStart).Hours() / 24
|
|
diskFrac := clamp(h.diskFrac+h.diskGrowthPerDay*days+r.NormFloat64()*0.002, 0.02, 0.985)
|
|
|
|
// A fixed boot moment per host, far enough back that uptimes read
|
|
// like real long-lived servers (and differ from each other).
|
|
bootOffset := time.Duration(3+int(phase*17)) * 24 * time.Hour
|
|
uptime := int64(t.Add(bootOffset).Sub(backfillStart).Seconds())
|
|
|
|
return newRecord(h, h.service, t, logsv1.Severity_SEVERITY_INFO, "host metrics", map[string]string{
|
|
"cairnobs.metrics": "true",
|
|
"cpu_percent": fmt.Sprintf("%.2f", cpu),
|
|
"mem_used_bytes": strconv.FormatInt(int64(float64(h.memTotal)*memFrac), 10),
|
|
"mem_total_bytes": strconv.FormatInt(h.memTotal, 10),
|
|
"disk_used_bytes": strconv.FormatInt(int64(float64(h.diskTot)*diskFrac), 10),
|
|
"disk_total_bytes": strconv.FormatInt(h.diskTot, 10),
|
|
"cpu_cores": strconv.Itoa(h.cores),
|
|
"os_name": h.os,
|
|
"kernel_version": h.kernel,
|
|
"arch": h.arch,
|
|
"uptime_seconds": strconv.FormatInt(uptime, 10),
|
|
"ipv4_addresses": h.ipv4,
|
|
"ipv6_addresses": h.ipv6,
|
|
})
|
|
}
|
|
|
|
func heartbeatRecord(h *host, t time.Time) *logsv1.LogRecord {
|
|
return newRecord(h, h.service, t, logsv1.Severity_SEVERITY_INFO, "agent heartbeat", map[string]string{
|
|
"cairnobs.heartbeat": "true",
|
|
"agent_version": h.agentVersion,
|
|
})
|
|
}
|