Give the demo a live synthetic fleet, dashboards, and alert rules

The demo had 75k generic records across eight host-0N/service pairs, one dashboard, one alert rule, and -- because nothing ever called AgentControl.CheckIn -- a completely empty Agents page. /hack/demo-simulator replaces the generic data with a fictional but coherent fleet: 14 hosts running nginx, an API tier, workers, Postgres, Redis, mail, Linux journals and Windows event logs, whose messages and attributes look like what those services actually write. It backfills a week (~370k records, ~20s) and then keeps running. Running continuously is the point, not an implementation detail. Three things the demo has to show are only true if data keeps arriving: the Agents page marks a host stale once check-ins stop, alert rules evaluate over trailing windows and would freeze in one state against a static dataset, and any "last 15 minutes" view is empty on data that stopped growing overnight. It also emits metrics/heartbeats and answers CheckIn faithfully enough that the remote-config editor's pending -> applied transition works end to end. Seeded incidents give the data something to find: an api-02 outage with matching slow queries on db-01, 5xx at the edge and cascading job failures; an SSH probe burst; a spam wave; a disk filling up; and one decommissioned host left deliberately stale. /hack/demo-seed holds the rest of the deployment -- the nightly reset, eight dashboards (64 panels, every viz type but line), eleven alert rules across three notification targets, and the systemd unit. Rule thresholds are calibrated against what the simulator actually produces: the first pass had four rules whose thresholds the traffic could never reach and one that fired during normal operation. No line charts: dashboard panels reject the raw-SQL escape hatch, and the pipe language has no time-bucketing, so a real time axis isn't expressible today. Noted in demo-seed/README.md rather than papered over.
2026-08-22 16:12:35 -07:00
parent e6a58f58ea
commit bcb9a01cd6
31 changed files with 3031 additions and 0 deletions
@@ -0,0 +1,167 @@
+package main
+
+import (
+	"context"
+	"log"
+	"sync"
+	"time"
+
+	agentv1 "github.com/cairnobs/cairnobs/proto/sentry/agent/v1"
+)
+
+// The Agents page is populated by the AgentControl.CheckIn RPC, not by
+// log volume: ingest's internal/agentregistry upserts one row per
+// (tenant, host) on every check-in, and nothing else ever writes that
+// table. A demo with plenty of logs but no check-ins therefore shows an
+// empty Agents page -- which is exactly the state this replaces.
+//
+// The simulated agents are faithful to the real protocol in the two
+// places a demo viewer can actually observe it:
+//
+//   - Remote config edits round-trip. The response's DesiredOverride
+//     version is echoed back as applied_override_version on the next
+//     check-in, so editing an agent's batch size or log paths in the web
+//     UI shows the real pending -> applied transition instead of a badge
+//     stuck on "pending" forever.
+//   - Restart commands are consumed. A queued RESTART is delivered
+//     at-most-once and cleared by ingest the moment it hands it out; the
+//     simulated agent acknowledges it by logging and resetting its
+//     applied-version state, the same visible outcome a real restart has.
+type simAgent struct {
+	h *host
+	// appliedVersion is what this agent reports having applied. Starts
+	// empty (never applied an override) and follows whatever the server
+	// hands back, one check-in behind -- the same one-tick lag a real
+	// agent has.
+	appliedVersion string
+	// applied is the override itself, kept so the *reported* config on
+	// subsequent check-ins reflects it. A real agent restarts its
+	// batcher/heartbeat with the new settings and then reports the new
+	// values back; without this the web UI would show a config marked
+	// "applied" next to reported values that never changed, which reads
+	// like the edit silently failed.
+	applied *agentv1.DesiredOverride
+}
+
+// reportedConfig is what the agent says it is currently running: its
+// local agent.toml settings, with any applied override layered on top.
+func (a *simAgent) reportedConfig() *agentv1.ReportedConfig {
+	cfg := &agentv1.ReportedConfig{
+		AgentVersion:         a.h.agentVersion,
+		SourceKind:           a.h.sourceKind,
+		SourceDetail:         a.h.sourceDetail,
+		BatchMaxSize:         uint64(a.h.batchMax),
+		BatchFlushIntervalMs: uint64(a.h.batchFlushMS),
+		HeartbeatEnabled:     true,
+		HeartbeatIntervalMs:  uint64(a.h.heartbeatMS),
+	}
+	if o := a.applied; o != nil {
+		if o.BatchMaxSize != nil {
+			cfg.BatchMaxSize = o.GetBatchMaxSize()
+		}
+		if o.BatchFlushIntervalMs != nil {
+			cfg.BatchFlushIntervalMs = o.GetBatchFlushIntervalMs()
+		}
+		if o.HeartbeatEnabled != nil {
+			cfg.HeartbeatEnabled = o.GetHeartbeatEnabled()
+		}
+		if o.HeartbeatIntervalMs != nil {
+			cfg.HeartbeatIntervalMs = o.GetHeartbeatIntervalMs()
+		}
+		// A journald-unit override changes what the agent is tailing,
+		// which is exactly what source_detail describes.
+		if o.JournaldUnit != nil && a.h.sourceKind == "journald" {
+			if u := o.GetJournaldUnit(); u != "" {
+				cfg.SourceDetail = "unit=" + u
+			} else {
+				cfg.SourceDetail = "whole journal"
+			}
+		}
+	}
+	return cfg
+}
+
+func (a *simAgent) checkIn(ctx context.Context, client agentv1.AgentControlClient) error {
+	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
+	defer cancel()
+
+	resp, err := client.CheckIn(ctx, &agentv1.CheckInRequest{
+		Host:                   a.h.name,
+		Service:                a.h.service,
+		CurrentConfig:          a.reportedConfig(),
+		AppliedOverrideVersion: a.appliedVersion,
+	})
+	if err != nil {
+		return err
+	}
+
+	if resp.GetHasOverride() && resp.GetOverride() != nil {
+		if v := resp.GetOverride().GetVersion(); v != a.appliedVersion {
+			log.Printf("agent %s: applying remote config override version %s", a.h.name, v)
+			a.appliedVersion = v
+			a.applied = resp.GetOverride()
+		}
+	} else if a.applied != nil {
+		// The override was cleared (DELETE /agents/{host}/config). A real
+		// agent falls back to its local agent.toml at that point, and so
+		// must this one -- otherwise the web UI shows an agent with no
+		// override still reporting the settings that override gave it,
+		// and the Clear button looks broken.
+		log.Printf("agent %s: remote config override cleared, reverting to local config", a.h.name)
+		a.applied = nil
+		a.appliedVersion = ""
+	}
+	if resp.GetPendingCommand() == agentv1.AgentCommand_AGENT_COMMAND_RESTART {
+		log.Printf("agent %s: restart command received, simulating restart", a.h.name)
+	}
+	return nil
+}
+
+// runCheckIns keeps every non-stale agent checking in on its own
+// heartbeat interval for as long as ctx lives. Stale hosts check in once
+// at startup and never again, which is what puts a genuine "stale" row
+// on the Agents page a few minutes into any demo session.
+func runCheckIns(ctx context.Context, client agentv1.AgentControlClient) {
+	var agents []*simAgent
+	for i := range fleet {
+		agents = append(agents, &simAgent{h: &fleet[i]})
+	}
+
+	var wg sync.WaitGroup
+	for _, a := range agents {
+		if err := a.checkIn(ctx, client); err != nil {
+			log.Printf("agent %s: initial check-in failed: %v", a.h.name, err)
+		}
+		if a.h.stale {
+			continue
+		}
+		wg.Add(1)
+		go func(a *simAgent) {
+			defer wg.Done()
+			ticker := time.NewTicker(time.Duration(a.h.heartbeatMS) * time.Millisecond)
+			defer ticker.Stop()
+			for {
+				select {
+				case <-ctx.Done():
+					return
+				case <-ticker.C:
+					if err := a.checkIn(ctx, client); err != nil && ctx.Err() == nil {
+						log.Printf("agent %s: check-in failed: %v", a.h.name, err)
+					}
+				}
+			}
+		}(a)
+	}
+	log.Printf("registered %d simulated agents (%d checking in every heartbeat interval)", len(agents), len(agents)-staleCount())
+	wg.Wait()
+}
+
+func staleCount() int {
+	n := 0
+	for i := range fleet {
+		if fleet[i].stale {
+			n++
+		}
+	}
+	return n
+}
@@ -0,0 +1,655 @@
+package main
+
+import (
+	"fmt"
+	"math/rand"
+	"strconv"
+	"strings"
+	"time"
+
+	logsv1 "github.com/cairnobs/cairnobs/proto/sentry/logs/v1"
+)
+
+// One generator per service. Each returns a record whose *message* reads
+// like the real thing that service writes (combined-log-format nginx
+// lines, Postgres duration/statement lines, Stalwart-shaped SMTP events,
+// sshd/UFW journald lines) and whose *attributes* carry the structured
+// fields those messages contain. Both halves matter: the message is what
+// free-text search (`message:"connection refused"`) matches, the
+// attributes are what `where status>=500` and `stats ... by path`
+// aggregate over, and a demo that only had one of them would leave half
+// the query language with nothing to show.
+
+var (
+	// Two IP pools, deliberately distinct: legitimate client traffic in
+	// documentation ranges, and a small set of "attacker" addresses that
+	// recur across sshd failures and UFW blocks, so a viewer who spots
+	// one in the Security dashboard can pivot on it and find the rest.
+	clientIPs = []string{
+		"203.0.113.14", "203.0.113.72", "203.0.113.109", "198.51.100.7",
+		"198.51.100.42", "192.0.2.28", "192.0.2.155", "203.0.113.201",
+	}
+	attackerIPs = []string{
+		"45.155.205.233", "185.191.171.12", "89.248.165.74", "141.98.11.60",
+	}
+
+	userAgents = []string{
+		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0 Safari/537.36",
+		"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:135.0) Gecko/20100101 Firefox/135.0",
+		"Mozilla/5.0 (iPhone; CPU iPhone OS 18_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.3 Mobile/15E148 Safari/604.1",
+		"curl/8.11.1",
+		"shop-mobile-android/4.12.0 (okhttp/4.12.0)",
+		"Googlebot/2.1 (+http://www.google.com/bot.html)",
+	}
+
+	// Routes are shared between the nginx and api generators on purpose:
+	// the same request shows up in the edge tier's access log and the
+	// application tier's own log, which is exactly what makes a
+	// "requests by path" panel comparable across services.
+	routes = []struct {
+		method string
+		path   string
+		route  string
+		weight int
+		slow   bool
+	}{
+		{"GET", "/", "/", 14, false},
+		{"GET", "/products", "/products", 12, false},
+		{"GET", "/products/%d", "/products/:id", 16, false},
+		{"GET", "/api/v1/cart", "/api/v1/cart", 9, false},
+		{"POST", "/api/v1/cart/items", "/api/v1/cart/items", 7, false},
+		{"POST", "/api/v1/checkout", "/api/v1/checkout", 5, true},
+		{"GET", "/api/v1/orders", "/api/v1/orders", 6, false},
+		{"GET", "/api/v1/orders/%d", "/api/v1/orders/:id", 5, false},
+		{"POST", "/api/v1/auth/login", "/api/v1/auth/login", 6, false},
+		{"GET", "/api/v1/search", "/api/v1/search", 8, true},
+		{"GET", "/static/app.%s.js", "/static/*", 10, false},
+		{"GET", "/healthz", "/healthz", 4, false},
+		{"POST", "/api/v1/webhooks/stripe", "/api/v1/webhooks/stripe", 3, false},
+	}
+	routeWeightTotal int
+
+	regions   = []string{"us-east-1", "us-west-2", "eu-west-1"}
+	appVerson = "[email protected]"
+)
+
+func init() {
+	for _, r := range routes {
+		routeWeightTotal += r.weight
+	}
+}
+
+func pickRoute(r *rand.Rand) (method, path, route string, slow bool) {
+	n := r.Intn(routeWeightTotal)
+	for _, rt := range routes {
+		if n -= rt.weight; n < 0 {
+			p := rt.path
+			switch {
+			case strings.Contains(p, "%d"):
+				p = fmt.Sprintf(p, 1000+r.Intn(9000))
+			case strings.Contains(p, "%s"):
+				p = fmt.Sprintf(p, hexString(r, 8))
+			}
+			return rt.method, p, rt.route, rt.slow
+		}
+	}
+	return "GET", "/", "/", false
+}
+
+func hexString(r *rand.Rand, n int) string {
+	const hexDigits = "0123456789abcdef"
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = hexDigits[r.Intn(16)]
+	}
+	return string(b)
+}
+
+func pick[T any](r *rand.Rand, xs []T) T { return xs[r.Intn(len(xs))] }
+
+func newRecord(h *host, service string, t time.Time, sev logsv1.Severity, msg string, attrs map[string]string) *logsv1.LogRecord {
+	return &logsv1.LogRecord{
+		TimestampUnixNano: t.UnixNano(),
+		Host:              h.name,
+		Service:           service,
+		Severity:          sev,
+		Message:           msg,
+		Attributes:        attrs,
+	}
+}
+
+// severityForStatus keeps the severity column and the status attribute
+// telling the same story -- a 500 that logged at INFO would make
+// `severity=ERROR` and `where status>=500` disagree, and a demo where
+// two obvious queries contradict each other is worse than one with less
+// data.
+func severityForStatus(status int) logsv1.Severity {
+	switch {
+	case status >= 500:
+		return logsv1.Severity_SEVERITY_ERROR
+	case status >= 400:
+		return logsv1.Severity_SEVERITY_WARN
+	default:
+		return logsv1.Severity_SEVERITY_INFO
+	}
+}
+
+func nginxRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord {
+	method, path, route, slow := pickRoute(r)
+	clientIP := pick(r, clientIPs)
+
+	// The edge tier mirrors whatever the API tier is doing: during the
+	// outage window a good share of upstream requests come back 5xx here
+	// too. Outside one it still isn't zero -- a real edge always returns
+	// the occasional 502 from an upstream restarting or a slow request
+	// tripping proxy_read_timeout, and a demo whose 5xx count is exactly
+	// zero for six days straight makes every 5xx panel and rule look
+	// broken rather than healthy.
+	upstreamErrRate := c.apiErrorRate
+	if upstreamErrRate == 0 {
+		upstreamErrRate = 0.008
+	}
+	status := 200
+	switch {
+	case r.Float64() < upstreamErrRate:
+		status = pick(r, []int{502, 503, 504})
+	case r.Float64() < 0.04:
+		status = pick(r, []int{404, 401, 403, 429})
+	case r.Float64() < 0.06:
+		status = 301
+	}
+
+	base := 40 + r.Float64()*180
+	if slow {
+		base *= 3
+	}
+	latency := base * c.latencyMult * (0.6 + r.Float64()*0.9)
+	bytes := 400 + r.Intn(60000)
+	referrer := "-"
+	if r.Float64() < 0.55 {
+		referrer = "https://shop.example.com" + pick(r, []string{"/", "/products", "/cart"})
+	}
+	ua := pick(r, userAgents)
+
+	msg := fmt.Sprintf(`%s - - [%s] "%s %s HTTP/1.1" %d %d %q %q %.3f`,
+		clientIP, t.UTC().Format("02/Jan/2006:15:04:05 -0700"),
+		method, path, status, bytes, referrer, ua, latency/1000)
+
+	attrs := map[string]string{
+		"remote_addr": clientIP,
+		"method":      method,
+		"path":        path,
+		"route":       route,
+		"status":      strconv.Itoa(status),
+		"bytes":       strconv.Itoa(bytes),
+		"latency_ms":  fmt.Sprintf("%.1f", latency),
+		"referrer":    referrer,
+		"user_agent":  ua,
+		"vhost":       "shop.example.com",
+	}
+
+	// A slice of edge-tier traffic is error-log lines rather than
+	// access-log ones -- the same thing a real nginx host ships from two
+	// files under one service.
+	if status >= 500 && r.Float64() < 0.5 {
+		upstream := fmt.Sprintf("10.0.2.%d:8080", 21+r.Intn(3))
+		msg = fmt.Sprintf(`%s [error] %d#0: *%d connect() failed (111: Connection refused) while connecting to upstream, client: %s, server: shop.example.com, request: "%s %s HTTP/1.1", upstream: "http://%s%s"`,
+			t.UTC().Format("2006/01/02 15:04:05"), 1000+r.Intn(900), r.Intn(90000), clientIP, method, path, upstream, path)
+		attrs["upstream_addr"] = upstream
+		attrs["log_kind"] = "error"
+	} else {
+		attrs["log_kind"] = "access"
+	}
+
+	return newRecord(h, "nginx", t, severityForStatus(status), msg, attrs)
+}
+
+var apiErrors = []struct {
+	code   string
+	detail string
+}{
+	{"db_pool_exhausted", "could not acquire a database connection: pool exhausted after 5000ms"},
+	{"upstream_timeout", "payment provider request timed out after 30s"},
+	{"null_reference", "unhandled exception in OrderService.finalize: nil pointer dereference"},
+	{"serialization_failure", "could not serialize access due to concurrent update"},
+	{"rate_limited_upstream", "inventory service returned 429, giving up after 3 retries"},
+}
+
+func apiRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord {
+	method, path, route, slow := pickRoute(r)
+	traceID := hexString(r, 16)
+	userID := fmt.Sprintf("u_%d", 1000+r.Intn(4000))
+	region := pick(r, regions)
+
+	errRate := c.apiErrorRate
+	if errRate == 0 {
+		errRate = 0.012 // the healthy baseline: a real service is never at exactly zero
+	}
+
+	status := 200
+	switch {
+	case r.Float64() < errRate:
+		status = pick(r, []int{500, 503})
+	case r.Float64() < 0.05:
+		status = pick(r, []int{400, 401, 404, 422, 429})
+	case method == "POST" && r.Float64() < 0.3:
+		status = 201
+	}
+
+	base := 25 + r.Float64()*120
+	if slow {
+		base *= 3.5
+	}
+	latency := base * c.latencyMult * (0.6 + r.Float64()*0.8)
+	dbTime := latency * (0.2 + r.Float64()*0.5)
+
+	attrs := map[string]string{
+		"method":     method,
+		"path":       path,
+		"route":      route,
+		"status":     strconv.Itoa(status),
+		"latency_ms": fmt.Sprintf("%.1f", latency),
+		"db_time_ms": fmt.Sprintf("%.1f", dbTime),
+		"trace_id":   traceID,
+		"user_id":    userID,
+		"region":     region,
+		"version":    appVerson,
+	}
+
+	var msg string
+	if status >= 500 {
+		e := pick(r, apiErrors)
+		attrs["error_code"] = e.code
+		msg = fmt.Sprintf("%s %s -> %d in %.0fms trace=%s: %s", method, path, status, latency, traceID, e.detail)
+	} else {
+		msg = fmt.Sprintf("%s %s -> %d in %.0fms trace=%s user=%s region=%s", method, path, status, latency, traceID, userID, region)
+	}
+	return newRecord(h, "api", t, severityForStatus(status), msg, attrs)
+}
+
+var workerJobs = []struct {
+	name  string
+	queue string
+	msMin int
+	msMax int
+}{
+	{"order.confirmation_email", "email", 120, 900},
+	{"report.daily_sales", "reports", 4000, 22000},
+	{"inventory.reconcile", "inventory", 800, 6000},
+	{"image.thumbnail", "media", 200, 2500},
+	{"search.reindex", "search", 3000, 30000},
+	{"webhook.retry", "webhooks", 100, 1500},
+}
+
+func workerRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord {
+	j := pick(r, workerJobs)
+	dur := float64(j.msMin+r.Intn(j.msMax-j.msMin)) * c.latencyMult
+	queueDepth := r.Intn(40)
+	attempt := 1
+	if r.Float64() < 0.12 {
+		attempt = 2 + r.Intn(2)
+	}
+
+	attrs := map[string]string{
+		"job":         j.name,
+		"queue":       j.queue,
+		"duration_ms": fmt.Sprintf("%.0f", dur),
+		"attempt":     strconv.Itoa(attempt),
+		"queue_depth": strconv.Itoa(queueDepth),
+	}
+
+	failRate := c.jobFailureRate
+	if failRate == 0 {
+		failRate = 0.05 // normal operation: retries exist because jobs do fail
+	}
+
+	switch {
+	case r.Float64() < failRate:
+		attrs["result"] = "failed"
+		reason := pick(r, []string{
+			"SMTP connection refused by relay",
+			"request timeout after 30s calling inventory-service",
+			"deadlock detected while updating orders",
+		})
+		return newRecord(h, "worker", t, logsv1.Severity_SEVERITY_ERROR,
+			fmt.Sprintf("job %s failed after %d attempts in %.0fms: %s", j.name, attempt, dur, reason), attrs)
+	case queueDepth > 32:
+		attrs["result"] = "ok"
+		return newRecord(h, "worker", t, logsv1.Severity_SEVERITY_WARN,
+			fmt.Sprintf("job %s completed in %.0fms but queue %s is backing up (depth=%d)", j.name, dur, j.queue, queueDepth), attrs)
+	default:
+		attrs["result"] = "ok"
+		return newRecord(h, "worker", t, logsv1.Severity_SEVERITY_INFO,
+			fmt.Sprintf("job %s completed in %.0fms (queue=%s attempt=%d)", j.name, dur, j.queue, attempt), attrs)
+	}
+}
+
+var pgStatements = []struct {
+	kind  string
+	table string
+	sql   string
+	msMin int
+	msMax int
+}{
+	{"SELECT", "orders", "SELECT o.*, c.email FROM orders o JOIN customers c ON c.id = o.customer_id WHERE o.customer_id = $1 ORDER BY o.created_at DESC LIMIT 50", 4, 120},
+	{"SELECT", "products", "SELECT * FROM products WHERE tsv @@ plainto_tsquery($1) LIMIT 100", 30, 900},
+	{"INSERT", "order_items", "INSERT INTO order_items (order_id, product_id, qty, price_cents) VALUES ($1, $2, $3, $4)", 2, 40},
+	{"UPDATE", "inventory", "UPDATE inventory SET on_hand = on_hand - $1 WHERE sku = $2", 3, 250},
+	{"SELECT", "sessions", "SELECT * FROM sessions WHERE token = $1", 1, 20},
+	{"DELETE", "carts", "DELETE FROM carts WHERE updated_at < now() - interval '30 days'", 200, 4000},
+}
+
+func postgresRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord {
+	// A minority of Postgres log volume is lifecycle/connection noise
+	// rather than statement logging, same as the real thing.
+	if r.Float64() < 0.18 {
+		switch r.Intn(3) {
+		case 0:
+			conns := 20 + r.Intn(90)
+			return newRecord(h, "postgres", t, logsv1.Severity_SEVERITY_INFO,
+				fmt.Sprintf("connection authorized: user=shop_app database=shop application_name=%s SSL enabled", appVerson),
+				map[string]string{"db": "shop", "db_user": "shop_app", "connections": strconv.Itoa(conns), "query_kind": "connect"})
+		case 1:
+			return newRecord(h, "postgres", t, logsv1.Severity_SEVERITY_INFO,
+				fmt.Sprintf("checkpoint complete: wrote %d buffers (%.1f%%); %d WAL file(s) added; sync=%.3f s, total=%.3f s",
+					800+r.Intn(4000), r.Float64()*8, r.Intn(4), r.Float64(), 1+r.Float64()*6),
+				map[string]string{"db": "shop", "query_kind": "checkpoint"})
+		default:
+			return newRecord(h, "postgres", t, logsv1.Severity_SEVERITY_WARN,
+				fmt.Sprintf("could not receive data from client: Connection reset by peer (pid=%d)", 2000+r.Intn(8000)),
+				map[string]string{"db": "shop", "query_kind": "connection_error", "pid": strconv.Itoa(2000 + r.Intn(8000))})
+		}
+	}
+
+	s := pick(r, pgStatements)
+	dur := float64(s.msMin+r.Intn(s.msMax-s.msMin)) * c.latencyMult
+	attrs := map[string]string{
+		"db":          "shop",
+		"db_user":     "shop_app",
+		"query_kind":  s.kind,
+		"table":       s.table,
+		"duration_ms": fmt.Sprintf("%.1f", dur),
+		"rows":        strconv.Itoa(r.Intn(500)),
+		"pid":         strconv.Itoa(2000 + r.Intn(8000)),
+	}
+	sev := logsv1.Severity_SEVERITY_DEBUG
+	if dur > 1000 {
+		sev = logsv1.Severity_SEVERITY_WARN
+	}
+	return newRecord(h, "postgres", t, sev,
+		fmt.Sprintf("duration: %.3f ms  statement: %s", dur, s.sql), attrs)
+}
+
+func redisRecord(h *host, t time.Time, r *rand.Rand, _ conditions) *logsv1.LogRecord {
+	used := int64(3<<30) + r.Int63n(2<<30)
+	clients := 40 + r.Intn(160)
+	attrs := map[string]string{
+		"used_memory_bytes": strconv.FormatInt(used, 10),
+		"connected_clients": strconv.Itoa(clients),
+	}
+	switch n := r.Intn(10); {
+	case n < 4:
+		attrs["op"] = "bgsave"
+		return newRecord(h, "redis", t, logsv1.Severity_SEVERITY_INFO,
+			fmt.Sprintf("Background saving terminated with success (%d changes in %d seconds)", 10000+r.Intn(50000), 60), attrs)
+	case n < 6:
+		evicted := 200 + r.Intn(4000)
+		attrs["op"] = "evict"
+		attrs["keys_evicted"] = strconv.Itoa(evicted)
+		return newRecord(h, "redis", t, logsv1.Severity_SEVERITY_WARN,
+			fmt.Sprintf("Evicted %d keys to stay under maxmemory (used_memory=%.1fGB)", evicted, float64(used)/float64(1<<30)), attrs)
+	case n < 8:
+		attrs["op"] = "client"
+		return newRecord(h, "redis", t, logsv1.Severity_SEVERITY_INFO,
+			fmt.Sprintf("Accepted %s:%d (connected_clients=%d)", pick(r, []string{"10.0.2.21", "10.0.2.22", "10.0.2.23", "10.0.3.31"}), 40000+r.Intn(20000), clients), attrs)
+	case n < 9:
+		attrs["op"] = "replication"
+		return newRecord(h, "redis", t, logsv1.Severity_SEVERITY_INFO,
+			"Synchronization with replica 10.0.4.52:6379 succeeded", attrs)
+	default:
+		attrs["op"] = "slowlog"
+		micros := 12000 + r.Intn(90000)
+		attrs["latency_ms"] = fmt.Sprintf("%.1f", float64(micros)/1000)
+		return newRecord(h, "redis", t, logsv1.Severity_SEVERITY_WARN,
+			fmt.Sprintf("slowlog entry: KEYS session:* took %d usec", micros), attrs)
+	}
+}
+
+var mailDomains = []string{"example.com", "example.net", "mail.example.org", "shop.example.com", "gmail.com", "outlook.com"}
+
+func smtpRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord {
+	queueID := hexString(r, 12)
+	remote := pick(r, clientIPs)
+	if c.spamWave || r.Float64() < 0.15 {
+		remote = pick(r, attackerIPs)
+	}
+	sender := pick(r, mailDomains)
+	rcpt := pick(r, []string{"shop.example.com", "cairnobs.example.com"})
+	size := 2000 + r.Intn(400000)
+
+	authFailRate := 0.06
+	spamRate := 0.08
+	if c.spamWave {
+		authFailRate = 0.45
+		spamRate = 0.35
+	}
+
+	attrs := map[string]string{
+		"queue_id":      queueID,
+		"remote_addr":   remote,
+		"sender_domain": sender,
+		"rcpt_domain":   rcpt,
+		"size_bytes":    strconv.Itoa(size),
+	}
+
+	switch {
+	case r.Float64() < authFailRate:
+		user := pick(r, []string{"admin", "postmaster", "info", "sales", "test"})
+		attrs["result"] = "auth_failed"
+		attrs["auth_user"] = user
+		return newRecord(h, "smtp", t, logsv1.Severity_SEVERITY_WARN,
+			fmt.Sprintf("auth-not-allowed: authentication failed for user %q from [%s] (mechanism=PLAIN)", user, remote), attrs)
+	case r.Float64() < spamRate:
+		score := 6 + r.Float64()*12
+		attrs["result"] = "spam_reject"
+		attrs["spam_score"] = fmt.Sprintf("%.1f", score)
+		return newRecord(h, "smtp", t, logsv1.Severity_SEVERITY_WARN,
+			fmt.Sprintf("spam-reject: message <%s@%s> from [%s] rejected, score %.1f above threshold 5.0", queueID, sender, remote, score), attrs)
+	case r.Float64() < 0.1:
+		attrs["result"] = "deferred"
+		return newRecord(h, "smtp", t, logsv1.Severity_SEVERITY_WARN,
+			fmt.Sprintf("deferred: <%s> to @%s temporarily rejected (450 4.2.1 mailbox busy), retry in 15m", queueID, rcpt), attrs)
+	default:
+		attrs["result"] = "delivered"
+		return newRecord(h, "smtp", t, logsv1.Severity_SEVERITY_INFO,
+			fmt.Sprintf("delivered: <%s> from @%s to @%s size=%d in %.2fs", queueID, sender, rcpt, size, 0.2+r.Float64()*3), attrs)
+	}
+}
+
+// internetFacing reports whether a host takes connections straight off
+// the internet. Only these see a probe window: an internal host behind
+// the edge tier keeps its ordinary background noise either way, and
+// pretending otherwise would show a brute-force burst arriving
+// simultaneously on hosts that aren't reachable at all.
+func internetFacing(h *host) bool { return internetFacingName(h.name) }
+
+func internetFacingName(name string) bool {
+	return name == "mail-01" || strings.HasPrefix(name, "edge-")
+}
+
+// systemRecord is the journald stream every Linux host ships: sshd, UFW,
+// systemd units, and the occasional kernel message. This is the stream
+// the Security dashboard and the brute-force/UFW alert rules read.
+func systemRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord {
+	pid := strconv.Itoa(400 + r.Intn(30000))
+
+	// During a probe window an internet-facing host's system stream is
+	// dominated by failed logins and firewall blocks.
+	probing := c.bruteForce && internetFacing(h)
+	roll := r.Float64()
+	if probing {
+		roll *= 0.35
+	}
+
+	switch {
+	case roll < 0.22:
+		src := pick(r, attackerIPs)
+		user := pick(r, []string{"admin", "root", "ubuntu", "oracle", "postgres", "git", "test"})
+		port := 40000 + r.Intn(20000)
+		return newRecord(h, "system", t, logsv1.Severity_SEVERITY_WARN,
+			fmt.Sprintf("sshd[%s]: Failed password for invalid user %s from %s port %d ssh2", pid, user, src, port),
+			map[string]string{
+				"unit": "ssh.service", "pid": pid, "remote_addr": src,
+				"ssh_user": user, "src_port": strconv.Itoa(port), "auth_result": "failed",
+			})
+	case roll < 0.4:
+		src := pick(r, attackerIPs)
+		dport := pick(r, []int{22, 23, 445, 3389, 5432, 6379, 8080, 3306})
+		sport := 40000 + r.Intn(20000)
+		return newRecord(h, "system", t, logsv1.Severity_SEVERITY_WARN,
+			fmt.Sprintf("kernel: [UFW BLOCK] IN=eth0 OUT= MAC=00:16:3e:%s SRC=%s DST=%s LEN=60 TOS=0x00 PREC=0x00 TTL=52 ID=%d PROTO=TCP SPT=%d DPT=%d WINDOW=1024 SYN",
+				hexString(r, 2)+":"+hexString(r, 2)+":"+hexString(r, 2), src, h.ipv4, r.Intn(65000), sport, dport),
+			map[string]string{
+				"unit": "kernel", "remote_addr": src, "ufw_action": "BLOCK",
+				"dst_port": strconv.Itoa(dport), "src_port": strconv.Itoa(sport), "proto": "TCP",
+			})
+	case roll < 0.52:
+		user := pick(r, []string{"john", "deploy", "ansible"})
+		src := pick(r, []string{"203.0.113.5", "10.0.0.9", "198.51.100.7"})
+		return newRecord(h, "system", t, logsv1.Severity_SEVERITY_INFO,
+			fmt.Sprintf("sshd[%s]: Accepted publickey for %s from %s port %d ssh2: ED25519 SHA256:%s", pid, user, src, 40000+r.Intn(20000), hexString(r, 20)),
+			map[string]string{
+				"unit": "ssh.service", "pid": pid, "remote_addr": src,
+				"ssh_user": user, "auth_result": "accepted",
+			})
+	case roll < 0.62:
+		user := pick(r, []string{"john", "deploy"})
+		cmd := pick(r, []string{"/usr/bin/systemctl restart shop-api.service", "/usr/bin/apt-get update", "/usr/bin/journalctl -u shop-worker -n 200"})
+		return newRecord(h, "system", t, logsv1.Severity_SEVERITY_INFO,
+			fmt.Sprintf("sudo: %s : TTY=pts/0 ; PWD=/home/%s ; USER=root ; COMMAND=%s", user, user, cmd),
+			map[string]string{"unit": "sudo", "ssh_user": user, "command": cmd})
+	case roll < 0.8:
+		unit := pick(r, []string{"logrotate.service", "apt-daily.service", "systemd-tmpfiles-clean.service", "fstrim.service"})
+		return newRecord(h, "system", t, logsv1.Severity_SEVERITY_INFO,
+			fmt.Sprintf("systemd[1]: %s: Deactivated successfully.", unit),
+			map[string]string{"unit": unit, "pid": "1"})
+	case roll < 0.92:
+		unit := pick(r, []string{"shop-api.service", "shop-worker.service", "nginx.service", "cairnobs-agent.service"})
+		return newRecord(h, "system", t, logsv1.Severity_SEVERITY_INFO,
+			fmt.Sprintf("systemd[1]: Reloaded %s.", unit),
+			map[string]string{"unit": unit, "pid": "1"})
+	case roll < 0.97:
+		return newRecord(h, "system", t, logsv1.Severity_SEVERITY_ERROR,
+			fmt.Sprintf("kernel: TCP: request_sock_TCP: Possible SYN flooding on port 443. Sending cookies. Check SNMP counters."),
+			map[string]string{"unit": "kernel", "dst_port": "443"})
+	default:
+		proc := pick(r, []string{"python3", "node", "ruby"})
+		return newRecord(h, "system", t, logsv1.Severity_SEVERITY_FATAL,
+			fmt.Sprintf("kernel: Out of memory: Killed process %s (%s) total-vm:%dkB, anon-rss:%dkB", pid, proc, 2000000+r.Intn(4000000), 1000000+r.Intn(3000000)),
+			map[string]string{"unit": "kernel", "pid": pid, "process": proc})
+	}
+}
+
+var winEvents = []struct {
+	id       string
+	provider string
+	channel  string
+	sev      logsv1.Severity
+	message  string
+	weight   int
+}{
+	{"4624", "Microsoft-Windows-Security-Auditing", "Security", logsv1.Severity_SEVERITY_INFO, "An account was successfully logged on.", 20},
+	{"4625", "Microsoft-Windows-Security-Auditing", "Security", logsv1.Severity_SEVERITY_WARN, "An account failed to log on.", 10},
+	{"4634", "Microsoft-Windows-Security-Auditing", "Security", logsv1.Severity_SEVERITY_INFO, "An account was logged off.", 14},
+	{"4688", "Microsoft-Windows-Security-Auditing", "Security", logsv1.Severity_SEVERITY_INFO, "A new process has been created.", 12},
+	{"4720", "Microsoft-Windows-Security-Auditing", "Security", logsv1.Severity_SEVERITY_WARN, "A user account was created.", 2},
+	{"4740", "Microsoft-Windows-Security-Auditing", "Security", logsv1.Severity_SEVERITY_WARN, "A user account was locked out.", 3},
+	{"7036", "Service Control Manager", "System", logsv1.Severity_SEVERITY_INFO, "The Windows Update service entered the running state.", 16},
+	{"7031", "Service Control Manager", "System", logsv1.Severity_SEVERITY_ERROR, "The SQL Server (MSSQLSERVER) service terminated unexpectedly.", 3},
+	{"1000", "Application Error", "Application", logsv1.Severity_SEVERITY_ERROR, "Faulting application name: ShopSync.exe, version 4.2.1.0, exception code 0xc0000005", 5},
+	{"6008", "EventLog", "System", logsv1.Severity_SEVERITY_ERROR, "The previous system shutdown was unexpected.", 1},
+	{"41", "Microsoft-Windows-Kernel-Power", "System", logsv1.Severity_SEVERITY_FATAL, "The system has rebooted without cleanly shutting down first.", 1},
+}
+
+var winWeightTotal int
+
+func init() {
+	for _, e := range winEvents {
+		winWeightTotal += e.weight
+	}
+}
+
+// eventlogRecord mirrors /hack/windows-fixture's attribute contract
+// (winevt.* keys) so a query written against one works against the
+// other -- that fixture stays the small correctness check it was built
+// as; this one supplies demo volume.
+func eventlogRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord {
+	n := r.Intn(winWeightTotal)
+	e := winEvents[0]
+	for _, cand := range winEvents {
+		if n -= cand.weight; n < 0 {
+			e = cand
+			break
+		}
+	}
+	// A probe window reaches the Windows tier as failed logons too.
+	if c.bruteForce && r.Float64() < 0.5 {
+		e = winEvents[1]
+	}
+
+	attrs := map[string]string{
+		"winevt.event_id":      e.id,
+		"winevt.provider":      e.provider,
+		"winevt.channel":       e.channel,
+		"winevt.computer":      h.name,
+		"winevt.record_number": strconv.Itoa(100000 + r.Intn(900000)),
+	}
+	msg := e.message
+	switch e.id {
+	case "4624", "4625", "4634":
+		user := pick(r, []string{"SHOP\\svc_sync", "SHOP\\jcoffey", "SHOP\\administrator", "SHOP\\backup"})
+		logonType := pick(r, []string{"3", "10", "2"})
+		attrs["winevt.target_user"] = user
+		attrs["winevt.logon_type"] = logonType
+		src := pick(r, clientIPs)
+		if e.id == "4625" {
+			src = pick(r, attackerIPs)
+			attrs["winevt.status"] = "0xC000006D"
+		}
+		attrs["remote_addr"] = src
+		msg = fmt.Sprintf("%s Account: %s  Logon Type: %s  Source Network Address: %s", e.message, user, logonType, src)
+	case "4688":
+		proc := pick(r, []string{"C:\\Windows\\System32\\cmd.exe", "C:\\Program Files\\ShopSync\\ShopSync.exe", "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe"})
+		attrs["winevt.process"] = proc
+		msg = fmt.Sprintf("%s New Process Name: %s", e.message, proc)
+	case "4740":
+		user := pick(r, []string{"SHOP\\jcoffey", "SHOP\\svc_sync"})
+		attrs["winevt.target_user"] = user
+		msg = fmt.Sprintf("%s Account Name: %s  Caller Computer Name: %s", e.message, user, h.name)
+	}
+	return newRecord(h, "eventlog", t, e.sev, msg, attrs)
+}
+
+// primaryRecord dispatches to whichever generator matches this host's
+// role. Kept as one switch rather than a func field on host so fleet.go
+// stays pure data.
+func primaryRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord {
+	switch h.service {
+	case "nginx":
+		return nginxRecord(h, t, r, c)
+	case "api":
+		return apiRecord(h, t, r, c)
+	case "worker":
+		return workerRecord(h, t, r, c)
+	case "postgres":
+		return postgresRecord(h, t, r, c)
+	case "redis":
+		return redisRecord(h, t, r, c)
+	case "smtp":
+		return smtpRecord(h, t, r, c)
+	case "eventlog":
+		return eventlogRecord(h, t, r, c)
+	default:
+		return newRecord(h, h.service, t, logsv1.Severity_SEVERITY_INFO, "heartbeat", nil)
+	}
+}
@@ -0,0 +1,241 @@
+package main
+
+// The synthetic fleet the demo deployment pretends to be monitoring: a
+// small e-commerce shop's infrastructure. Every host here is fictional,
+// but the shape is deliberately realistic -- an edge/nginx tier, a
+// three-node API tier, background workers, one Postgres, one Redis, one
+// Stalwart mail host, two Windows boxes, and one decommissioned host
+// left behind on purpose so the Agents page has a genuinely stale row to
+// show (see stale below).
+//
+// One entry here is one *host*, not one agent process: the `agents`
+// table is UNIQUE (tenant_id, host) (see
+// metadata/migrations/0037_create_agents.sql), so a host maps to exactly
+// one agent row, one metrics series, and one heartbeat stream. Log
+// records are not bound by that -- a host emits its primary service's
+// logs plus, on Linux, the journald `system` stream every real
+// deployment also collects.
+
+type host struct {
+	name    string
+	service string // primary log service, and the service its agent reports
+
+	// Static context the Hosts page shows alongside utilization (see
+	// web/src/lib/api.ts's HostMetrics) -- a viewer can't judge "21% CPU"
+	// without the core count, or "is this normal" without uptime.
+	os       string
+	kernel   string
+	arch     string
+	cores    int
+	memTotal int64
+	diskTot  int64
+	ipv4     string
+	ipv6     string
+
+	// Utilization baselines. Each sample wanders around these rather
+	// than being redrawn independently, so the Hosts page shows a host
+	// with a personality (a busy API node, an idle cache) instead of the
+	// same noise everywhere.
+	cpuBase  float64 // mean CPU percent
+	memFrac  float64 // mean fraction of memTotal in use
+	diskFrac float64 // fraction of diskTot in use at the START of the backfill window
+	// diskGrowthPerDay pushes diskFrac up over the window -- the "disk
+	// slowly filling up" story the disk-usage alert rule fires on. Zero
+	// for every host that isn't part of that story.
+	diskGrowthPerDay float64
+
+	// Peak-hour log rates, in events per minute, before the diurnal
+	// curve and -rate-scale are applied. systemPerMin is the journald
+	// system stream (sshd/ufw/systemd/kernel), zero on Windows hosts.
+	eventsPerMin float64
+	systemPerMin float64
+
+	// What this host's agent reports about itself on CheckIn.
+	agentVersion string
+	sourceKind   string // "journald", "file", "eventlog"
+	sourceDetail string
+	batchMax     int64
+	batchFlushMS int64
+	heartbeatMS  int64
+
+	// stale hosts check in exactly once at startup and then go quiet, so
+	// the Agents page's staleness heuristic (last_seen older than 3x the
+	// heartbeat interval, floor 5 minutes -- see
+	// web/src/routes/agents/+page.svelte) flags them a few minutes into
+	// any demo session. They emit no logs and no metrics either: a host
+	// whose agent is gone stops producing everything, not just
+	// heartbeats.
+	stale bool
+}
+
+const agentVersion = "0.6.2"
+
+var fleet = []host{
+	{
+		name: "edge-01", service: "nginx",
+		os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64",
+		cores: 4, memTotal: 8 << 30, diskTot: 100 << 30,
+		ipv4: "10.0.1.11", ipv6: "2600:3c02::f03c:94ff:fe1a:1101",
+		cpuBase: 22, memFrac: 0.41, diskFrac: 0.36,
+		eventsPerMin: 15, systemPerMin: 0.7,
+		agentVersion: agentVersion, sourceKind: "file", sourceDetail: "/var/log/nginx/access.log",
+		batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000,
+	},
+	{
+		name: "edge-02", service: "nginx",
+		os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64",
+		cores: 4, memTotal: 8 << 30, diskTot: 100 << 30,
+		ipv4: "10.0.1.12", ipv6: "2600:3c02::f03c:94ff:fe1a:1102",
+		cpuBase: 19, memFrac: 0.38, diskFrac: 0.33,
+		eventsPerMin: 13, systemPerMin: 0.6,
+		agentVersion: agentVersion, sourceKind: "file", sourceDetail: "/var/log/nginx/access.log",
+		batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000,
+	},
+	{
+		name: "api-01", service: "api",
+		os: "Debian GNU/Linux 13 (trixie)", kernel: "6.12.9-amd64", arch: "x86_64",
+		cores: 8, memTotal: 16 << 30, diskTot: 160 << 30,
+		ipv4: "10.0.2.21", ipv6: "2600:3c02::f03c:94ff:fe1a:2101",
+		cpuBase: 34, memFrac: 0.52, diskFrac: 0.29,
+		eventsPerMin: 10, systemPerMin: 0.5,
+		agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=shop-api.service",
+		batchMax: 1000, batchFlushMS: 3000, heartbeatMS: 60000,
+	},
+	{
+		name: "api-02", service: "api",
+		os: "Debian GNU/Linux 13 (trixie)", kernel: "6.12.9-amd64", arch: "x86_64",
+		cores: 8, memTotal: 16 << 30, diskTot: 160 << 30,
+		ipv4: "10.0.2.22", ipv6: "2600:3c02::f03c:94ff:fe1a:2102",
+		cpuBase: 37, memFrac: 0.57, diskFrac: 0.31,
+		eventsPerMin: 10, systemPerMin: 0.5,
+		agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=shop-api.service",
+		batchMax: 1000, batchFlushMS: 3000, heartbeatMS: 60000,
+	},
+	{
+		name: "api-03", service: "api",
+		// One host deliberately a release behind, so the Agents page's
+		// agent_version column shows a fleet that isn't uniformly
+		// upgraded -- the normal state of any real fleet.
+		os: "Debian GNU/Linux 12 (bookworm)", kernel: "6.1.0-25-amd64", arch: "x86_64",
+		cores: 4, memTotal: 8 << 30, diskTot: 160 << 30,
+		ipv4: "10.0.2.23", ipv6: "2600:3c02::f03c:94ff:fe1a:2103",
+		cpuBase: 41, memFrac: 0.61, diskFrac: 0.44,
+		eventsPerMin: 9, systemPerMin: 0.5,
+		agentVersion: "0.5.4", sourceKind: "journald", sourceDetail: "unit=shop-api.service",
+		batchMax: 1000, batchFlushMS: 3000, heartbeatMS: 60000,
+	},
+	{
+		name: "worker-01", service: "worker",
+		os: "Debian GNU/Linux 13 (trixie)", kernel: "6.12.9-amd64", arch: "x86_64",
+		cores: 4, memTotal: 8 << 30, diskTot: 200 << 30,
+		ipv4: "10.0.3.31", ipv6: "2600:3c02::f03c:94ff:fe1a:3101",
+		cpuBase: 46, memFrac: 0.63, diskFrac: 0.4,
+		eventsPerMin: 4.5, systemPerMin: 0.4,
+		agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=shop-worker.service",
+		batchMax: 1000, batchFlushMS: 5000, heartbeatMS: 60000,
+	},
+	{
+		name: "worker-02", service: "worker",
+		os: "Debian GNU/Linux 13 (trixie)", kernel: "6.12.9-amd64", arch: "x86_64",
+		cores: 4, memTotal: 8 << 30, diskTot: 200 << 30,
+		ipv4: "10.0.3.32", ipv6: "2600:3c02::f03c:94ff:fe1a:3102",
+		cpuBase: 52, memFrac: 0.71, diskFrac: 0.62,
+		// The one host with a real, visible trend: ~4 points of disk a
+		// day, so a 7-day backfill window ends with it close to full and
+		// the "Disk filling up" alert rule has something true to fire on.
+		diskGrowthPerDay: 0.04,
+		eventsPerMin:     4.5, systemPerMin: 0.4,
+		agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=shop-worker.service",
+		batchMax: 1000, batchFlushMS: 5000, heartbeatMS: 60000,
+	},
+	{
+		name: "db-01", service: "postgres",
+		os: "Debian GNU/Linux 13 (trixie)", kernel: "6.12.9-amd64", arch: "x86_64",
+		cores: 8, memTotal: 32 << 30, diskTot: 500 << 30,
+		ipv4: "10.0.4.41", ipv6: "2600:3c02::f03c:94ff:fe1a:4101",
+		cpuBase: 28, memFrac: 0.74, diskFrac: 0.51,
+		eventsPerMin: 6, systemPerMin: 0.4,
+		agentVersion: agentVersion, sourceKind: "file", sourceDetail: "/var/log/postgresql/postgresql-17-main.log",
+		batchMax: 1000, batchFlushMS: 5000, heartbeatMS: 60000,
+	},
+	{
+		name: "cache-01", service: "redis",
+		os: "Ubuntu 22.04.5 LTS", kernel: "5.15.0-118-generic", arch: "x86_64",
+		cores: 2, memTotal: 8 << 30, diskTot: 50 << 30,
+		ipv4: "10.0.4.51", ipv6: "2600:3c02::f03c:94ff:fe1a:5101",
+		cpuBase: 11, memFrac: 0.58, diskFrac: 0.18,
+		eventsPerMin: 2, systemPerMin: 0.3,
+		agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=redis-server.service",
+		batchMax: 500, batchFlushMS: 10000, heartbeatMS: 60000,
+	},
+	{
+		name: "mail-01", service: "smtp",
+		os: "Debian GNU/Linux 13 (trixie)", kernel: "6.12.9-amd64", arch: "x86_64",
+		cores: 2, memTotal: 4 << 30, diskTot: 250 << 30,
+		ipv4: "198.51.100.25", ipv6: "2600:3c06::2000:7dff:fe55:2501",
+		cpuBase: 14, memFrac: 0.46, diskFrac: 0.57,
+		eventsPerMin: 6, systemPerMin: 1.2, // internet-facing: more scan/ssh noise than an internal host
+		agentVersion: agentVersion, sourceKind: "file", sourceDetail: "/opt/stalwart/logs/current.log",
+		batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000,
+	},
+	{
+		name: "arm-build-01", service: "worker",
+		// The fleet's one non-x86 host, so `stats count by arch`-style
+		// questions and the Hosts page's Architecture row have more than
+		// one answer in them.
+		os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "aarch64",
+		cores: 8, memTotal: 16 << 30, diskTot: 120 << 30,
+		ipv4: "10.0.5.61", ipv6: "2600:3c02::f03c:94ff:fe1a:6101",
+		cpuBase: 63, memFrac: 0.55, diskFrac: 0.47,
+		eventsPerMin: 3, systemPerMin: 0.3,
+		agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=buildkite-agent.service",
+		batchMax: 1000, batchFlushMS: 5000, heartbeatMS: 60000,
+	},
+	{
+		name: "WIN-APP-01", service: "eventlog",
+		os: "Windows Server 2022 Datacenter", kernel: "10.0.20348", arch: "x86_64",
+		cores: 4, memTotal: 16 << 30, diskTot: 250 << 30,
+		ipv4: "10.0.6.71", ipv6: "",
+		cpuBase: 26, memFrac: 0.64, diskFrac: 0.42,
+		eventsPerMin: 2.5, systemPerMin: 0,
+		agentVersion: agentVersion, sourceKind: "eventlog", sourceDetail: "channels=Security,System,Application",
+		batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000,
+	},
+	{
+		name: "WIN-SQL-01", service: "eventlog",
+		os: "Windows Server 2019 Standard", kernel: "10.0.17763", arch: "x86_64",
+		cores: 8, memTotal: 32 << 30, diskTot: 500 << 30,
+		ipv4: "10.0.6.72", ipv6: "",
+		cpuBase: 33, memFrac: 0.78, diskFrac: 0.66,
+		eventsPerMin: 2, systemPerMin: 0,
+		agentVersion: "0.5.4", sourceKind: "eventlog", sourceDetail: "channels=Security,System,Application",
+		batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000,
+	},
+	{
+		name: "legacy-01", service: "nginx",
+		os: "Ubuntu 20.04.6 LTS", kernel: "5.4.0-192-generic", arch: "x86_64",
+		cores: 2, memTotal: 4 << 30, diskTot: 40 << 30,
+		ipv4: "10.0.1.19", ipv6: "",
+		cpuBase: 3, memFrac: 0.22, diskFrac: 0.71,
+		eventsPerMin: 0, systemPerMin: 0,
+		agentVersion: "0.4.9", sourceKind: "file", sourceDetail: "/var/log/nginx/access.log",
+		batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000,
+		stale: true,
+	},
+}
+
+// linuxHosts is every host whose agent tails journald or a file -- i.e.
+// everything that also produces the `system` service stream. Windows
+// hosts produce eventlog records instead, and the stale host produces
+// nothing at all.
+func linuxHosts() []*host {
+	var out []*host
+	for i := range fleet {
+		h := &fleet[i]
+		if h.stale || h.service == "eventlog" {
+			continue
+		}
+		out = append(out, h)
+	}
+	return out
+}
@@ -0,0 +1,18 @@
+module github.com/cairnobs/cairnobs/hack/demo-simulator
+
+go 1.25.0
+
+replace github.com/cairnobs/cairnobs/proto => ../../proto
+
+require (
+	github.com/cairnobs/cairnobs/proto v0.0.0-00010101000000-000000000000
+	google.golang.org/grpc v1.83.0
+)
+
+require (
+	golang.org/x/net v0.55.0 // indirect
+	golang.org/x/sys v0.45.0 // indirect
+	golang.org/x/text v0.39.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect
+	google.golang.org/protobuf v1.36.12 // indirect
+)
@@ -0,0 +1,38 @@
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
+github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
+go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
+go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
+go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
+go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
+go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
+go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
+go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
+go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI=
+go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
+go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
+go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
+golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
+golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
+golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
+golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus=
+golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM=
+gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
+gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
+google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ=
+google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ=
+google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
+google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
@@ -0,0 +1,134 @@
+package main
+
+import (
+	"math"
+	"time"
+)
+
+// Incidents are what turn a wall of uniform noise into a dataset worth
+// clicking around in: a demo viewer who filters to 5xx, or opens the
+// Alerts page, should find a *story* (one bad API node, a burst of SSH
+// probing, a spam wave) rather than the same flat error rate everywhere.
+// Every window below is expressed relative to `origin` -- the moment the
+// simulator started, which is also the end of the backfill window -- so
+// a freshly reset demo always has its incidents at the same, recent,
+// predictable offsets no matter what day it is.
+type conditions struct {
+	// apiErrorRate replaces the API tier's baseline 5xx probability for
+	// the affected host during an outage window.
+	apiErrorRate float64
+	// latencyMult multiplies API/DB latencies -- an outage that only
+	// changed status codes without slowing anything down wouldn't look
+	// like a real one.
+	latencyMult float64
+	// jobFailureRate replaces the worker tier's baseline failure
+	// probability. An API/database outage doesn't stay in the request
+	// path: the same failing dependencies take background jobs down with
+	// them, which is also the only thing that ever makes the job-failure
+	// alert rule true -- a steady 5% background failure rate is normal
+	// operation, not an incident.
+	jobFailureRate float64
+	// bruteForce and spamWave switch the system/smtp generators from
+	// their normal mix to an attack-shaped one for the window.
+	bruteForce bool
+	spamWave   bool
+}
+
+// Windows, as offsets back from origin. Kept as one table so the story
+// is readable in one place and the alert rules in
+// /hack/demo-seed/alerts can be written against known-true conditions.
+const (
+	apiOutageStart = 8 * time.Hour
+	apiOutageEnd   = 6*time.Hour + 30*time.Minute
+	apiOutageHost  = "api-02"
+
+	bruteForceStart = 14 * time.Hour
+	bruteForceEnd   = 13 * time.Hour
+
+	spamWaveStart = 30 * time.Hour
+	spamWaveEnd   = 26 * time.Hour
+
+	// Live mode can't rely on the backfill windows above -- they recede
+	// into the past as a demo session runs. These recurring bursts keep
+	// the *present* interesting too, which is what the alert rules
+	// (evaluating over -5m/-10m windows) actually see: without them,
+	// every rule would settle into a permanent OK state a few minutes
+	// after a reset and the Alerts page would never do anything again.
+	liveErrorBurstPeriod = 47 * time.Minute
+	liveErrorBurstLen    = 5 * time.Minute
+	liveProbePeriod      = 2 * time.Hour
+	liveProbeLen         = 8 * time.Minute
+	liveSpamPeriod       = 3 * time.Hour
+	liveSpamLen          = 10 * time.Minute
+)
+
+func conditionsAt(t, origin time.Time, h *host) conditions {
+	hostName := h.name
+	c := conditions{latencyMult: 1}
+	since := origin.Sub(t)
+
+	// Is an API-tier outage in effect? Two sources feed the same
+	// handling: the backfilled incident window, and -- in live mode --
+	// the recurring burst that keeps the present interesting.
+	outage := 0.0
+	if since <= apiOutageStart && since >= apiOutageEnd {
+		outage = 0.42
+	}
+	bruteForce := since <= bruteForceStart && since >= bruteForceEnd
+	spamWave := since <= spamWaveStart && since >= spamWaveEnd
+
+	if t.After(origin) {
+		// Phases are measured from origin so the first burst of each kind
+		// lands a predictable few minutes into a demo session rather than
+		// immediately at reset.
+		elapsed := t.Sub(origin)
+		if phase := (elapsed + 10*time.Minute) % liveErrorBurstPeriod; phase < liveErrorBurstLen {
+			outage = 0.45
+		}
+		if phase := (elapsed + 20*time.Minute) % liveProbePeriod; phase < liveProbeLen {
+			bruteForce = true
+		}
+		if phase := (elapsed + 35*time.Minute) % liveSpamPeriod; phase < liveSpamLen {
+			spamWave = true
+		}
+	}
+
+	if outage > 0 {
+		switch {
+		case hostName == apiOutageHost:
+			c.apiErrorRate = outage
+			c.latencyMult = 4.5
+		case internetFacingName(hostName):
+			// The edge tier fronts all three API nodes, so roughly a
+			// third of what it proxies during the outage hits the failing
+			// one. Without this the outage would be invisible from the
+			// edge, which isn't how a viewer expects to be able to trace
+			// it: client-visible 5xx are exactly what makes it an outage
+			// rather than an internal blip.
+			c.apiErrorRate = outage / 3
+			c.latencyMult = 2
+		case hostName == "db-01":
+			// The database is the *cause*, not a second unrelated
+			// incident -- a viewer who drills from the API errors into
+			// the same window on db-01 should find slow queries waiting.
+			c.latencyMult = 6
+		case h.service == "worker":
+			c.jobFailureRate = 0.35
+			c.latencyMult = 2.5
+		}
+	}
+	c.bruteForce = bruteForce
+	c.spamWave = spamWave
+
+	return c
+}
+
+// diurnal scales event rates by time of day: a shop's traffic peaks
+// mid-afternoon UTC and bottoms out around 04:00, roughly a 3.5x spread.
+// Without this every chart is a flat line and the "last 24h" view tells
+// a viewer nothing that "last 1h" didn't.
+func diurnal(t time.Time) float64 {
+	hour := float64(t.UTC().Hour()) + float64(t.UTC().Minute())/60
+	// Peak at 15:00 UTC, trough at 03:00.
+	return 0.35 + 0.65*(0.5+0.5*math.Cos((hour-15)/24*2*math.Pi))
+}
@@ -0,0 +1,410 @@
+// Command demo-simulator is the demo deployment's whole synthetic world
+// in one process: a fictional fleet (see fleet.go) whose agents check in
+// over AgentControl, report CPU/memory/disk, and ship realistically
+// shaped logs for eight services -- backfilled across a window of
+// history first, then continuously in real time for as long as it runs.
+//
+// Why one long-running process rather than another one-shot fixture:
+// three of the demo's features are only convincing if data keeps
+// arriving. The Agents page marks a host stale once it stops checking in
+// (a one-shot fixture's fleet would go stale minutes after the nightly
+// reset); alert rules evaluate over trailing windows like -5m and would
+// settle into a permanent OK state against a frozen dataset; and a live
+// tail or a "last 15 minutes" dashboard over a dataset that stopped
+// growing at 03:00 shows an empty screen. Backfill alone can't fix any
+// of those.
+//
+// It does not replace /hack/benchmark-fixture (volume benchmarking) or
+// /hack/windows-fixture (Windows pipeline correctness) -- those stay the
+// focused tools they were built as. This one is for the demo.
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"flag"
+	"fmt"
+	"log"
+	"math/rand"
+	"os"
+	"os/signal"
+	"sort"
+	"sync"
+	"sync/atomic"
+	"syscall"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+
+	agentv1 "github.com/cairnobs/cairnobs/proto/sentry/agent/v1"
+	logsv1 "github.com/cairnobs/cairnobs/proto/sentry/logs/v1"
+)
+
+// metricsInterval is how often each host samples CPU/memory/disk and
+// emits a heartbeat, in both backfill and live mode -- matching the real
+// agent's default 60s heartbeat would multiply backfill volume for no
+// visible gain on a chart, so history is sampled more coarsely than the
+// present.
+const (
+	backfillMetricsInterval = 5 * time.Minute
+	liveMetricsInterval     = time.Minute
+	liveTick                = 5 * time.Second
+)
+
+func main() {
+	addr := flag.String("addr", "localhost:4317", "ingest gRPC address")
+	caFile := flag.String("ca", "../dev-certs/out/ca.pem", "CA cert path")
+	certFile := flag.String("cert", "../dev-certs/out/client.pem", "client cert path")
+	keyFile := flag.String("key", "../dev-certs/out/client-key.pem", "client key path")
+	backfill := flag.Duration("backfill", 168*time.Hour, "how much history to generate before going live; 0 skips backfill")
+	live := flag.Bool("live", true, "after backfill, keep generating events in real time until terminated")
+	rateScale := flag.Float64("rate-scale", 0.5, "multiplier on every host's per-minute event rate -- the knob for how much total data a backfill produces")
+	batchSize := flag.Int("batch-size", 1000, "records per PushBatch call")
+	concurrency := flag.Int("concurrency", 4, "concurrent PushBatch calls in flight during backfill")
+	seed := flag.Int64("seed", 0, "random seed; 0 uses the current time")
+	dryRun := flag.Bool("dry-run", false, "generate the backfill without connecting to ingest and print what it would have sent, then exit")
+	flag.Parse()
+
+	if *seed == 0 {
+		*seed = time.Now().UnixNano()
+	}
+
+	if *dryRun {
+		runDryRun(context.Background(), time.Now(), *backfill, *rateScale, *seed)
+		return
+	}
+
+	tlsConf, err := loadTLSConfig(*caFile, *certFile, *keyFile)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "loading TLS config:", err)
+		os.Exit(1)
+	}
+
+	conn, err := grpc.NewClient(*addr, grpc.WithTransportCredentials(credentials.NewTLS(tlsConf)))
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "dialing ingest:", err)
+		os.Exit(1)
+	}
+	defer conn.Close()
+
+	logs := logsv1.NewLogIngestClient(conn)
+	control := agentv1.NewAgentControlClient(conn)
+
+	// origin is both the end of the backfill window and the reference
+	// point every incident window is measured back from, so history and
+	// live traffic tell one continuous story.
+	origin := time.Now()
+
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+
+	if *backfill > 0 {
+		runBackfill(ctx, logs, origin, *backfill, *rateScale, *batchSize, *concurrency, *seed)
+	}
+	if ctx.Err() != nil {
+		return
+	}
+	if !*live {
+		return
+	}
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() { defer wg.Done(); runCheckIns(ctx, control) }()
+	// metricsAnchor is the oldest moment this run's metrics series
+	// covers. It has to be the same value in both modes: disk growth and
+	// uptime are both measured from it, and anchoring live samples at
+	// `origin` instead would make worker-02's disk usage jump backwards
+	// (and every uptime reset to near zero) the instant backfill ended.
+	metricsAnchor := origin.Add(-*backfill)
+	go func() { defer wg.Done(); runLive(ctx, logs, origin, metricsAnchor, *rateScale, *seed) }()
+	wg.Wait()
+	log.Println("demo-simulator stopped")
+}
+
+// runBackfill walks the history window a minute at a time, streaming
+// batches to a small pool of pushers as it goes rather than building the
+// whole dataset in memory first -- the demo box this runs on has 3GB of
+// RAM and a week of history is hundreds of thousands of records.
+func runBackfill(ctx context.Context, client logsv1.LogIngestClient, origin time.Time, window time.Duration, rateScale float64, batchSize, concurrency int, seed int64) {
+	start := origin.Add(-window)
+	log.Printf("backfilling %s of history (%s .. %s) at rate-scale %.2f",
+		window, start.UTC().Format(time.RFC3339), origin.UTC().Format(time.RFC3339), rateScale)
+
+	batches := make(chan []*logsv1.LogRecord, concurrency*2)
+	var sent atomic.Int64
+	var failed atomic.Int64
+
+	var pushers sync.WaitGroup
+	for i := 0; i < concurrency; i++ {
+		pushers.Add(1)
+		go func(worker int) {
+			defer pushers.Done()
+			for batch := range batches {
+				n, err := push(ctx, client, fmt.Sprintf("demo-backfill-%d-%d", worker, sent.Load()), batch)
+				if err != nil {
+					if ctx.Err() == nil {
+						log.Printf("backfill PushBatch failed: %v", err)
+					}
+					failed.Add(int64(len(batch)))
+					continue
+				}
+				total := sent.Add(int64(n))
+				if total%50000 < int64(batchSize) {
+					log.Printf("backfill: %d records sent", total)
+				}
+			}
+		}(i)
+	}
+
+	batch := make([]*logsv1.LogRecord, 0, batchSize)
+	walkHistory(ctx, origin, window, rateScale, seed, func(rec *logsv1.LogRecord) {
+		batch = append(batch, rec)
+		if len(batch) >= batchSize {
+			batches <- batch
+			batch = make([]*logsv1.LogRecord, 0, batchSize)
+		}
+	})
+	if len(batch) > 0 {
+		batches <- batch
+	}
+	close(batches)
+	pushers.Wait()
+
+	if f := failed.Load(); f > 0 {
+		log.Printf("backfill complete: %d records sent, %d dropped by failed pushes", sent.Load(), f)
+		return
+	}
+	log.Printf("backfill complete: %d records sent", sent.Load())
+}
+
+// walkHistory replays the backfill window a minute at a time, handing
+// every generated record to emit. Shared by the real backfill and
+// -dry-run so the two can never disagree about what a run would produce.
+func walkHistory(ctx context.Context, origin time.Time, window time.Duration, rateScale float64, seed int64, emit func(*logsv1.LogRecord)) {
+	start := origin.Add(-window)
+	rng := rand.New(rand.NewSource(seed))
+
+	nextMetrics := start
+	for minute := start; minute.Before(origin) && ctx.Err() == nil; minute = minute.Add(time.Minute) {
+		metricsDue := !minute.Before(nextMetrics)
+		if metricsDue {
+			nextMetrics = minute.Add(backfillMetricsInterval)
+		}
+		for i := range fleet {
+			h := &fleet[i]
+			if h.stale {
+				continue
+			}
+			c := conditionsAt(minute, origin, h)
+			for _, rec := range minuteRecords(h, minute, rng, c, rateScale) {
+				emit(rec)
+			}
+			if metricsDue {
+				emit(metricsRecord(h, minute, start, rng))
+				emit(heartbeatRecord(h, minute))
+			}
+		}
+	}
+}
+
+// runDryRun generates a backfill without sending it anywhere and reports
+// what it would have produced -- the volume/mix tuning knob, so
+// -rate-scale and the per-host rates in fleet.go can be adjusted without
+// pushing a few hundred thousand records into ClickHouse to find out.
+func runDryRun(ctx context.Context, origin time.Time, window time.Duration, rateScale float64, seed int64) {
+	byService := map[string]int{}
+	bySeverity := map[string]int{}
+	total := 0
+	walkHistory(ctx, origin, window, rateScale, seed, func(rec *logsv1.LogRecord) {
+		total++
+		byService[rec.GetService()]++
+		bySeverity[rec.GetSeverity().String()]++
+	})
+
+	fmt.Printf("dry run: %d records over %s at rate-scale %.2f (%.0f/min average)\n",
+		total, window, rateScale, float64(total)/window.Minutes())
+	fmt.Println("by service:")
+	for _, k := range sortedKeys(byService) {
+		fmt.Printf("  %-10s %8d\n", k, byService[k])
+	}
+	fmt.Println("by severity:")
+	for _, k := range sortedKeys(bySeverity) {
+		fmt.Printf("  %-22s %8d\n", k, bySeverity[k])
+	}
+}
+
+func sortedKeys(m map[string]int) []string {
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	return keys
+}
+
+// minuteRecords generates one host's events for one minute of wall
+// clock: its primary service's traffic, plus the journald `system`
+// stream every Linux host also ships.
+func minuteRecords(h *host, minute time.Time, rng *rand.Rand, c conditions, rateScale float64) []*logsv1.LogRecord {
+	shape := diurnal(minute) * rateScale
+	var out []*logsv1.LogRecord
+	for i := 0; i < countFor(h.eventsPerMin*shape, rng); i++ {
+		out = append(out, primaryRecord(h, jitter(minute, rng), rng, c))
+	}
+	if h.systemPerMin > 0 {
+		// System/journald volume rises during a probe window -- that
+		// burst is the whole point of the Security dashboard's panels.
+		sysRate := h.systemPerMin
+		if c.bruteForce && internetFacing(h) {
+			sysRate *= 12
+		}
+		for i := 0; i < countFor(sysRate*shape, rng); i++ {
+			out = append(out, systemRecord(h, jitter(minute, rng), rng, c))
+		}
+	}
+	return out
+}
+
+// countFor turns a fractional per-minute rate into a whole number of
+// events, carrying the fraction as a probability so a host rated at 0.4
+// events/min really does produce roughly two events every five minutes
+// instead of none at all.
+func countFor(rate float64, rng *rand.Rand) int {
+	n := int(rate)
+	if rng.Float64() < rate-float64(n) {
+		n++
+	}
+	return n
+}
+
+func jitter(minute time.Time, rng *rand.Rand) time.Time {
+	return minute.Add(time.Duration(rng.Int63n(int64(time.Minute))))
+}
+
+// runLive keeps the present moving: the same generators, driven by a
+// ticker instead of a cursor, so trailing-window alert rules, the Agents
+// page's staleness heuristic, and any "last 15 minutes" view all have
+// something real to read.
+func runLive(ctx context.Context, client logsv1.LogIngestClient, origin, metricsAnchor time.Time, rateScale float64, seed int64) {
+	log.Printf("live mode: generating events every %s", liveTick)
+	rng := rand.New(rand.NewSource(seed + 1))
+
+	// Fractional carry per host: at a 5-second tick most hosts are owed
+	// less than one event per tick, and dropping that remainder every
+	// time would silently zero out every low-rate stream.
+	carry := make(map[string]float64, len(fleet)*2)
+	ticker := time.NewTicker(liveTick)
+	defer ticker.Stop()
+	metricsTicker := time.NewTicker(liveMetricsInterval)
+	defer metricsTicker.Stop()
+
+	tickFraction := liveTick.Minutes()
+	last := time.Now()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+
+		case now := <-ticker.C:
+			var batch []*logsv1.LogRecord
+			for i := range fleet {
+				h := &fleet[i]
+				if h.stale {
+					continue
+				}
+				c := conditionsAt(now, origin, h)
+				shape := diurnal(now) * rateScale * tickFraction
+
+				n := carried(carry, h.name+"/primary", h.eventsPerMin*shape)
+				for j := 0; j < n; j++ {
+					batch = append(batch, primaryRecord(h, between(last, now, rng), rng, c))
+				}
+				if h.systemPerMin > 0 {
+					sysRate := h.systemPerMin
+					if c.bruteForce && internetFacing(h) {
+						sysRate *= 12
+					}
+					n := carried(carry, h.name+"/system", sysRate*shape)
+					for j := 0; j < n; j++ {
+						batch = append(batch, systemRecord(h, between(last, now, rng), rng, c))
+					}
+				}
+			}
+			last = now
+			if len(batch) == 0 {
+				continue
+			}
+			if _, err := push(ctx, client, fmt.Sprintf("demo-live-%d", now.Unix()), batch); err != nil && ctx.Err() == nil {
+				log.Printf("live PushBatch failed: %v", err)
+			}
+
+		case now := <-metricsTicker.C:
+			var batch []*logsv1.LogRecord
+			for i := range fleet {
+				h := &fleet[i]
+				if h.stale {
+					continue
+				}
+				batch = append(batch, metricsRecord(h, now, metricsAnchor, rng), heartbeatRecord(h, now))
+			}
+			if _, err := push(ctx, client, fmt.Sprintf("demo-metrics-%d", now.Unix()), batch); err != nil && ctx.Err() == nil {
+				log.Printf("metrics PushBatch failed: %v", err)
+			}
+		}
+	}
+}
+
+// carried accumulates a fractional event count for one stream until it
+// crosses 1, then spends the whole part. The leftover is kept, not
+// rounded away, so long-run volume matches the configured rate exactly
+// rather than drifting low -- at a 5-second tick most streams are owed
+// well under one event per tick, and rounding would zero them out.
+func carried(carry map[string]float64, key string, rate float64) int {
+	total := carry[key] + rate
+	n := int(total)
+	carry[key] = total - float64(n)
+	return n
+}
+
+func between(from, to time.Time, rng *rand.Rand) time.Time {
+	span := to.Sub(from)
+	if span <= 0 {
+		return to
+	}
+	return from.Add(time.Duration(rng.Int63n(int64(span))))
+}
+
+func push(ctx context.Context, client logsv1.LogIngestClient, batchID string, records []*logsv1.LogRecord) (int, error) {
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	resp, err := client.PushBatch(ctx, &logsv1.PushBatchRequest{BatchId: batchID, Records: records})
+	if err != nil {
+		return 0, err
+	}
+	return int(resp.GetAccepted()), nil
+}
+
+func loadTLSConfig(caFile, certFile, keyFile string) (*tls.Config, error) {
+	caPEM, err := os.ReadFile(caFile)
+	if err != nil {
+		return nil, fmt.Errorf("reading CA cert %s: %w", caFile, err)
+	}
+	caPool := x509.NewCertPool()
+	if !caPool.AppendCertsFromPEM(caPEM) {
+		return nil, fmt.Errorf("no valid certificates found in %s", caFile)
+	}
+
+	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		return nil, fmt.Errorf("loading client cert/key: %w", err)
+	}
+
+	return &tls.Config{
+		RootCAs:      caPool,
+		Certificates: []tls.Certificate{cert},
+	}, nil
+}
@@ -0,0 +1,85 @@
+package main
+
+import (
+	"fmt"
+	"hash/fnv"
+	"math"
+	"math/rand"
+	"strconv"
+	"time"
+
+	logsv1 "github.com/cairnobs/cairnobs/proto/sentry/logs/v1"
+)
+
+// Metrics records carry the exact attribute contract the Hosts page
+// reads (see web/src/lib/api.ts's getHostMetrics/listMetricsHosts): a
+// `cairnobs.metrics=true` tag plus utilization and static-context
+// fields, shipped as an ordinary tagged LogRecord because the query
+// language maps any non-standard field name to attributes['field'] with
+// automatic numeric casting -- no separate metrics pipeline exists, by
+// design. Heartbeat records use the same trick with
+// `cairnobs.heartbeat=true`, which is what the absence-style alert rules
+// watch for.
+
+// phaseOf gives each host its own deterministic offset into the wander
+// functions below, so two hosts with the same baseline don't move in
+// lockstep across the fleet's charts.
+func phaseOf(name string) float64 {
+	h := fnv.New32a()
+	_, _ = h.Write([]byte(name))
+	return float64(h.Sum32()%1000) / 1000 * 2 * math.Pi
+}
+
+func clamp(v, lo, hi float64) float64 {
+	return math.Max(lo, math.Min(hi, v))
+}
+
+// metricsRecord samples host h at time t. backfillStart anchors the
+// disk-growth trend so a host that's "filling up" is at its lowest at
+// the oldest end of the window and its highest right now, in whichever
+// order the samples happen to be generated.
+func metricsRecord(h *host, t, backfillStart time.Time, r *rand.Rand) *logsv1.LogRecord {
+	phase := phaseOf(h.name)
+	mins := float64(t.Unix()) / 60
+
+	// Two sine components of different periods plus noise: a slow
+	// business-hours swell and a faster one, so a CPU chart looks like a
+	// machine doing work rather than a random walk.
+	cpu := h.cpuBase * (1 +
+		0.45*math.Sin(mins/97+phase) +
+		0.2*math.Sin(mins/13+phase*2))
+	cpu = clamp(cpu*diurnal(t)+r.NormFloat64()*2.5, 0.4, 99)
+
+	memFrac := clamp(h.memFrac*(1+0.08*math.Sin(mins/211+phase))+r.NormFloat64()*0.01, 0.03, 0.97)
+
+	days := t.Sub(backfillStart).Hours() / 24
+	diskFrac := clamp(h.diskFrac+h.diskGrowthPerDay*days+r.NormFloat64()*0.002, 0.02, 0.985)
+
+	// A fixed boot moment per host, far enough back that uptimes read
+	// like real long-lived servers (and differ from each other).
+	bootOffset := time.Duration(3+int(phase*17)) * 24 * time.Hour
+	uptime := int64(t.Add(bootOffset).Sub(backfillStart).Seconds())
+
+	return newRecord(h, h.service, t, logsv1.Severity_SEVERITY_INFO, "host metrics", map[string]string{
+		"cairnobs.metrics": "true",
+		"cpu_percent":      fmt.Sprintf("%.2f", cpu),
+		"mem_used_bytes":   strconv.FormatInt(int64(float64(h.memTotal)*memFrac), 10),
+		"mem_total_bytes":  strconv.FormatInt(h.memTotal, 10),
+		"disk_used_bytes":  strconv.FormatInt(int64(float64(h.diskTot)*diskFrac), 10),
+		"disk_total_bytes": strconv.FormatInt(h.diskTot, 10),
+		"cpu_cores":        strconv.Itoa(h.cores),
+		"os_name":          h.os,
+		"kernel_version":   h.kernel,
+		"arch":             h.arch,
+		"uptime_seconds":   strconv.FormatInt(uptime, 10),
+		"ipv4_addresses":   h.ipv4,
+		"ipv6_addresses":   h.ipv6,
+	})
+}
+
+func heartbeatRecord(h *host, t time.Time) *logsv1.LogRecord {
+	return newRecord(h, h.service, t, logsv1.Severity_SEVERITY_INFO, "agent heartbeat", map[string]string{
+		"cairnobs.heartbeat": "true",
+		"agent_version":      h.agentVersion,
+	})
+}