diff --git a/docs/demo.md b/docs/demo.md index 21a08ca..bc59013 100644 --- a/docs/demo.md +++ b/docs/demo.md @@ -46,6 +46,11 @@ obs-beacond and the simulator, so nothing in it can reach anything else: - `fleet config -demo` writes the server's pieces; `fleet run -demo` runs the fleet (add `-ibmi 4` for IBM i and the HMC). Start the fleet first so its stand-ins are up for obs-beacond's first readings. +- `fleet run -history -state-key ` + first writes the last 30 days into obs-beacond's (SQLite) database, so + the graphs, reports and incident history aren't empty on a fresh demo + (below). It takes under a minute, before the stand-ins start, so wait + for them that long; it leaves a database that already has history alone. - The root is read-only and the state is in tmpfs, so restarting the container is a fresh demo; restart it on a timer (weekly is plenty) so history doesn't pile up. @@ -59,3 +64,14 @@ problems that come and go through the day, so incidents open, resolve and show up under Recently resolved. Because the plan runs on the clock, two visitors an hour apart see different problems, and a restarted demo shows the same ones the old one would have. + +The demo starts with a month behind it, worked out from the same plan and +clock (`scripts/fleet/history.go`): every host's hourly CPU, memory and +disks (a working day in its site's time zone, quieter weekends, memory +that creeps until a weekly restart, disks that fill until a clean-up), the +clusters' nodes (and a week of their controllers and VMs, to keep the +database small), the arrays' I/O, the IBM i partitions' CPU, the +Vaults' leases, the mail monitor's rounds, availability for the reports, +and an incident for each time a problem came and went, with notes on the +stories saying what the cause was. Its incident numbers carry on from +there. diff --git a/docs/development.md b/docs/development.md index 688ec24..100c6ef 100644 --- a/docs/development.md +++ b/docs/development.md @@ -82,6 +82,15 @@ restarted demo carries on where the clock is. Incidents open and resolve through the day, and a dozen or two are open at any moment. The mail server is healthy, to show what normal looks like. +What a host uses is a function of its name and the clock +(`scripts/fleet/load.go`), not a dice roll, so `-history +-state-key ` can write the last 30 days (`-history-days`) before +anything reports (`history.go`): hosts' and workloads' hours, mail rounds, +statuses, and an incident for each past spell of the problems in +`pastProblems` (scenario.go), whose keys and titles are obs-beacond's +own. When you change what a spell's problem says in obs-beacond, change +its line there too. + ## Dependencies and notices THIRD_PARTY_NOTICES.md lists every third-party component inside the diff --git a/internal/state/history.go b/internal/state/history.go index fd1819b..18e6915 100644 --- a/internal/state/history.go +++ b/internal/state/history.go @@ -245,3 +245,53 @@ func (d *DB) Prune(before time.Time) error { } return nil } + +// PutHostHours is PutHostHour for many, in one transaction: a backfill +// (the demo's 30 days, scripts/fleet) or an import. +func (d *DB) PutHostHours(hs []HostHour) error { + t, err := d.begin() + if err != nil { + return err + } + defer t.Rollback() + for _, h := range hs { + b, _ := json.Marshal(h.Disks) + if _, err := t.Exec(`INSERT INTO host_hours (host, hour, cpu, cpu_max, mem, mem_max, disks) VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (host, hour) DO UPDATE SET cpu=excluded.cpu, cpu_max=excluded.cpu_max, mem=excluded.mem, mem_max=excluded.mem_max, disks=excluded.disks`, + h.Host, h.Hour.Unix(), h.CPU, h.CPUMax, h.Mem, h.MemMax, string(b)); err != nil { + return err + } + } + return t.Commit() +} + +// PutMailRounds is PutMailRound for many, in one transaction. +func (d *DB) PutMailRounds(ms []MailRound) error { + t, err := d.begin() + if err != nil { + return err + } + defer t.Rollback() + for _, m := range ms { + if _, err := t.Exec(`INSERT INTO mail_rounds (at, seconds, queue) VALUES (?, ?, ?) + ON CONFLICT (at) DO UPDATE SET seconds=excluded.seconds, queue=excluded.queue`, m.At.Unix(), m.Seconds, m.Queue); err != nil { + return err + } + } + return t.Commit() +} + +// LogStatuses is LogStatus for many, in one transaction. +func (d *DB) LogStatuses(cs []Change) error { + t, err := d.begin() + if err != nil { + return err + } + defer t.Rollback() + for _, c := range cs { + if _, err := t.Exec(`INSERT INTO status_log (entity, at, status) VALUES (?, ?, ?)`, c.Entity, c.At.Unix(), c.Status); err != nil { + return err + } + } + return t.Commit() +} diff --git a/scripts/fleet/history.go b/scripts/fleet/history.go new file mode 100644 index 0000000..5c7a731 --- /dev/null +++ b/scripts/fleet/history.go @@ -0,0 +1,292 @@ +package main + +import ( + "fmt" + "log" + "math" + "slices" + "time" + + "github.com/cairnobs/cairnobs/internal/report" + "github.com/cairnobs/cairnobs/internal/state" +) + +// The demo's past (#39): before anything reports, fleet run -history +// writes the last 30 days into obs-beacond's database, worked out from the +// same clock the live demo runs on (load.go, scenario.go), so the graphs +// have their days and the Alerts page and reports have their incidents: +// +// - each host's hours: CPU and memory averages and peaks, disks; +// - each cluster's workloads' hours, an array's I/O and space, an IBM i +// partition's CPU, a Vault's leases; +// - the mail monitor's rounds; +// - hosts' and checks' statuses, from the start; +// - an incident each time a past problem (pastProblems) came and went, +// with a note from whoever looked at the stories. + +// pastFlaps are the statuses that change with a spell, for the reports' +// availability: an entity takes status while it's on, and after when +// it's over. +var pastFlaps = []struct { + entity string + s spell + status, after string +}{ + {"check/web-legacy", spellLegacy502, "warning", "good"}, + {"check/rbl-mx2", spellBlocklisted, "warning", "good"}, + {"svc/sites", spellLegacy502, "warning", "good"}, + {"svc/mail", spellBlocklisted, "warning", "good"}, + {"svc/postgres", spellReplicaBehind, "warning", "good"}, + {"svc/haproxy", spellReplicaBehind, "warning", "good"}, + // One copy of something there are several of is the service degraded, + // not down (topology.go, redundant parts). + {"svc/apps", spellAppExits, "warning", "good"}, + {"svc/redis", spellRedisFails, "warning", "good"}, +} + +// pastFleet is what the history is written for. +type pastFleet struct { + hosts []*host + clusters []*fakeCluster + vaults bool // the Vault stand-ins run + mail bool // the mail stand-in runs + checks []string +} + +func writeHistory(path, keyFile string, days int, f pastFleet, now time.Time) error { + start := time.Now() + db, err := state.Open(path, keyFile) + if err != nil { + return err + } + defer db.Close() + if !db.HistoryStart().IsZero() { + log.Printf("fleet: %s already has history; left as it is", path) + return nil + } + from := now.Truncate(time.Hour).Add(-time.Duration(days) * 24 * time.Hour) + to := now.Truncate(time.Hour) // the hour now is obs-beacond's to write + + var hs []state.HostHour + var statuses []state.Change + for _, h := range f.hosts { + name := h.r.Host + statuses = append(statuses, state.Change{Entity: "host/" + name, At: from, Status: "good"}) + for t := from; t.Before(to); t = t.Add(time.Hour) { + hs = append(hs, hostHour(h, t)) + } + } + for _, id := range f.checks { + statuses = append(statuses, state.Change{Entity: "check/" + id, At: from, Status: "good"}) + } + if f.checks != nil { // the demo's services (demoTopology) + for _, id := range []string{"shop", "api", "sites", "mail", "apps", "haproxy", "postgres", "redis", "nats"} { + statuses = append(statuses, state.Change{Entity: "svc/" + id, At: from, Status: "good"}) + } + // Flexible SSL on legacy.example is a lasting warning, and the + // shop is behind Cloudflare. + statuses = append(statuses, state.Change{Entity: "svc/cloudflare", At: from, Status: "warning"}, + state.Change{Entity: "svc/shop", At: from.Add(time.Second), Status: "warning"}) + for _, fl := range pastFlaps { + s := fl.s + for b := s.began(from).Add(s.every); b.Add(s.on).Before(now); b = b.Add(s.every) { + statuses = append(statuses, state.Change{Entity: fl.entity, At: b.Add(20 * time.Second), Status: fl.status}, + state.Change{Entity: fl.entity, At: b.Add(s.on + 20*time.Second), Status: fl.after}) + } + } + slices.SortStableFunc(statuses, func(a, b state.Change) int { return a.At.Compare(b.At) }) + } + if err := db.PutHostHours(hs); err != nil { + return fmt.Errorf("host hours: %w", err) + } + if err := db.LogStatuses(statuses); err != nil { + return fmt.Errorf("statuses: %w", err) + } + + nw := 0 + // Thousands of controllers and VMs get a week: a month of each is most + // of the database, and the demo's state is a 256 MB tmpfs. + few := to.Add(-time.Duration(min(days, 7)) * 24 * time.Hour) + for _, c := range f.clusters { + ws := clusterHours(c, from, few, to) + if err := db.PutWorkloadHours(ws); err != nil { + return fmt.Errorf("%s: %w", c.name, err) + } + nw += len(ws) + } + if f.vaults { + var ws []state.WorkloadHour + for name, base := range vaultLeaseBase { + for t := from; t.Before(to); t = t.Add(time.Hour) { + avg, top := 0.0, 0.0 + for m := range 4 { + l := float64(vaultLeases(base, t.Add(time.Duration(m)*15*time.Minute))) + avg += l / 4 + top = max(top, l) + } + ws = append(ws, state.WorkloadHour{Source: report.SourceKey("obs-beacond", name), Workload: name, Hour: t, Kind: report.KindCluster, + Leases: math.Round(avg), LeasesMax: top}) + } + } + if err := db.PutWorkloadHours(ws); err != nil { + return fmt.Errorf("vault: %w", err) + } + nw += len(ws) + } + + var ms []state.MailRound + if f.mail { + for t := from; t.Before(now.Add(-time.Minute)); t = t.Add(time.Minute) { + k := int(t.Unix() / 60) + ms = append(ms, state.MailRound{At: t, Seconds: 1.5 + float64(k*7919%4500)/1000, Queue: 3 + k%6}) + } + if err := db.PutMailRounds(ms); err != nil { + return fmt.Errorf("mail rounds: %w", err) + } + } + + n, err := writePastIncidents(db, from, now) + if err != nil { + return fmt.Errorf("incidents: %w", err) + } + log.Printf("fleet: wrote %d days of history in %s: %d host hours, %d workload hours, %d mail rounds, %d incidents", + days, time.Since(start).Round(time.Millisecond), len(hs), nw, len(ms), n) + return nil +} + +// hostHour is one host's hour, from a reading every five minutes, as +// obs-beacond's own (from every report) would roughly be. +func hostHour(h *host, t time.Time) state.HostHour { + name := h.r.Host + out := state.HostHour{Host: name, Hour: t} + const n = 12 + for k := range n { + at := t.Add(time.Duration(k)*5*time.Minute + 2*time.Minute) + c, m := cpuNow(name, at), memAt(name, at) + out.CPU += c / n + out.Mem += m / n + out.CPUMax = max(out.CPUMax, c*(1.1+0.3*seed(name, at.Format("15")))) + out.MemMax = max(out.MemMax, m+1.5) + } + out.CPU, out.CPUMax = round1(out.CPU), round1(min(out.CPUMax, 100)) + out.Mem, out.MemMax = round1(out.Mem), round1(min(out.MemMax, 100)) + end := t.Add(time.Hour - time.Minute) + for i, d := range h.r.Disks { + out.Disks = append(out.Disks, state.DiskSample{Mount: d.Mount, Used: uint64(float64(d.Size) * diskAt(name, i, end)), Total: d.Size}) + } + return out +} + +func round1(f float64) float64 { return math.Round(f*10) / 10 } + +// clusterHours is a cluster's workloads' hours, from two readings an hour, +// keyed the way obs-beacond keeps them (its historyKey): from from for +// the cluster itself and its nodes, from few for the rest. +func clusterHours(c *fakeCluster, from, few, to time.Time) []state.WorkloadHour { + if c.noUsage && c.usageFn == nil { + return nil + } + src := report.SourceKey(c.name, c.source) + c.mu.Lock() + names := map[string]report.Workload{} + for id, r := range c.recs { + names[id] = report.Workload{Namespace: r.Namespace, Name: r.Name, Kind: r.Kind} + } + c.mu.Unlock() + var out []state.WorkloadHour + for t := from; t.Before(to); t = t.Add(time.Hour) { + sum := map[string]*state.WorkloadHour{} + const n = 2 + for k := range n { + at := t.Add(time.Duration(k)*30*time.Minute + 10*time.Minute) + us := c.usageFn + if us == nil { + us = c.usage + } + for _, u := range us(at) { + r, ok := names[u.ID] + if !ok || t.Before(few) && r.Kind != report.KindNode && r.Kind != report.KindCluster { + continue + } + key := r.Name + if r.Namespace != "" { + key = r.Namespace + "/" + r.Name + } + h := sum[key] + if h == nil { + h = &state.WorkloadHour{Source: src, Workload: key, Hour: t, Kind: r.Kind} + sum[key] = h + } + h.CPU += u.CPU / n + h.Mem += float64(u.Mem) / n + h.CPUMax, h.MemMax = max(h.CPUMax, u.CPU), max(h.MemMax, float64(u.Mem)) + h.CPUOf, h.MemOf = u.CPUOf, float64(u.MemOf) + if io := u.IO; io != nil { + h.ReadOps += io.ReadOps / n + h.WriteOps += io.WriteOps / n + h.ReadUs += io.ReadUs / n + h.WriteUs += io.WriteUs / n + h.ReadBps += io.ReadBps / n + h.WriteBps += io.WriteBps / n + h.ReadUsMax, h.WriteUsMax = max(h.ReadUsMax, io.ReadUs*1.4), max(h.WriteUsMax, io.WriteUs*1.4) + h.Used, h.UsedOf = float64(io.Used), float64(io.UsedOf) + } + } + } + for _, h := range sum { + out = append(out, *h) + } + } + return out +} + +// writePastIncidents writes an incident each time a past problem came and +// went in [from, now), numbered in order from B-0001, and tells +// obs-beacond where the numbers carry on. +func writePastIncidents(db *state.DB, from, now time.Time) (int, error) { + type occ struct { + p past + began time.Time + first bool // the first of its kind on its day: it gets the note + } + var all []occ + for _, p := range pastProblems { + if p.s.every == 0 { + continue + } + day := "" + for b := p.s.began(from).Add(p.s.every); b.Add(p.s.on).Before(now); b = b.Add(p.s.every) { + d := b.UTC().Format("2006-01-02") + all = append(all, occ{p, b, d != day}) + day = d + } + } + slices.SortStableFunc(all, func(a, b occ) int { return a.began.Compare(b.began) }) + for k, o := range all { + p := o.p + // Seen within a reading or two, told after alerts.for (2m), and + // resolved within a minute of its end. + opened := o.began.Add(time.Duration(10+seedInt(p.key, o.began, 40)) * time.Second) + fired := opened.Add(2 * time.Minute) + resolved := o.began.Add(p.s.on + time.Duration(15+seedInt(p.key+" end", o.began, 45))*time.Second) + inc := state.Incident{ID: fmt.Sprintf("B-%04d", k+1), Key: p.key, Severity: p.sev, Title: p.title, Detail: p.detail, + Host: p.host, Href: p.href, Opened: opened, Fired: fired, Notified: fired, Resolved: resolved} + if p.sev == "critical" && p.s.on > 10*time.Minute { + inc.AckBy, inc.AckAt = "On call", fired.Add(time.Duration(2+seedInt(p.key+" ack", o.began, 6))*time.Minute) + } + if err := db.PutIncident(inc); err != nil { + return 0, err + } + if p.note != "" && o.first { + if err := db.SetNote(inc.ID, state.Note{Text: p.note, By: "On call", At: resolved.Add(20 * time.Minute)}); err != nil { + return 0, err + } + } + } + return len(all), db.SetMeta("incident_seq", fmt.Sprint(len(all))) +} + +// seedInt is 0 … n-1, fixed for a key and a moment. +func seedInt(key string, t time.Time, n int) int { + return int((jitter(key, t) + 1) / 2 * float64(n-1)) +} diff --git a/scripts/fleet/history_test.go b/scripts/fleet/history_test.go new file mode 100644 index 0000000..cee0f4c --- /dev/null +++ b/scripts/fleet/history_test.go @@ -0,0 +1,88 @@ +package main + +import ( + "encoding/base64" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/cairnobs/cairnobs/internal/state" +) + +// Hosts stay under the warnings (disks 80%, memory 90%), but fleet-0023's +// log volume, which crosses 80% for a while every ten days (#40). +func TestLoadStaysCalm(t *testing.T) { + now := time.Date(2026, 11, 16, 5, 0, 0, 0, time.UTC) + over := 0 + for i := range 150 { + name := hostName(i) + for h := range 30 * 24 { + at := now.Add(-time.Duration(h) * time.Hour) + if c := cpuNow(name, at); c <= 0 || c > 100 { + t.Fatalf("%s at %s: CPU %.1f", name, at, c) + } + if m := memAt(name, at); m >= 90 { + t.Fatalf("%s at %s: memory %.1f", name, at, m) + } + for k := range 2 { + if d := diskAt(name, k, at); d >= 0.8 { + if i != 23 || k != 1 { + t.Fatalf("%s disk %d at %s: %.2f", name, k, at, d) + } + over++ + } + } + } + } + if over == 0 || over > 30*24/3 { + t.Errorf("fleet-0023's log volume is over 80%% for %d of 720 hours", over) + } + // Not all alike, as every disk at 33% was. + if diskAt(hostName(1), 0, now) == diskAt(hostName(2), 0, now) { + t.Error("two hosts' disks are the same") + } +} + +func TestWriteHistory(t *testing.T) { + dir := t.TempDir() + key := filepath.Join(dir, "state.key") + if err := os.WriteFile(key, []byte(base64.StdEncoding.EncodeToString(make([]byte, 32))), 0o600); err != nil { + t.Fatal(err) + } + db := filepath.Join(dir, "beacon.db") + hosts := []*host{newHost(1, hostName(1), 2, 10*time.Second), newHost(2, hostName(2), 2, 10*time.Second)} + now := time.Now() + if err := writeHistory(db, key, 3, pastFleet{hosts: hosts, vaults: true, mail: true, checks: []string{"web-legacy"}}, now); err != nil { + t.Fatal(err) + } + d, err := state.Open(db, key) + if err != nil { + t.Fatal(err) + } + defer d.Close() + hs, _ := d.HostHours(now.Add(-4*24*time.Hour), now) + if len(hs) != 2*3*24 { + t.Errorf("%d host hours", len(hs)) + } + incs, _ := d.IncidentsBetween(now.Add(-4*24*time.Hour), now) + if len(incs) < 20 || d.Meta("incident_seq") != fmt.Sprint(len(incs)) { + t.Errorf("%d incidents, incident_seq %s", len(incs), d.Meta("incident_seq")) + } + for _, i := range incs { + if i.Resolved.After(now) || !i.Resolved.After(i.Opened) { + t.Fatalf("%s %s: opened %s, resolved %s", i.ID, i.Key, i.Opened, i.Resolved) + } + } + if d.HistoryStart().IsZero() { + t.Error("no statuses") + } + // A second run leaves it alone. + if err := writeHistory(db, key, 3, pastFleet{hosts: hosts}, now); err != nil { + t.Fatal(err) + } + if hs2, _ := d.HostHours(now.Add(-4*24*time.Hour), now); len(hs2) != len(hs) { + t.Errorf("written twice: %d hours", len(hs2)) + } +} diff --git a/scripts/fleet/load.go b/scripts/fleet/load.go new file mode 100644 index 0000000..fc4e829 --- /dev/null +++ b/scripts/fleet/load.go @@ -0,0 +1,112 @@ +package main + +import ( + "hash/fnv" + "math" + "time" +) + +// What a fake host uses, worked out from its name and the clock alone, so +// the live reports and the 30 days written before them (history.go) +// agree: a working day in its site's time zone and quieter weekends, +// memory that creeps on some hosts until a weekly restart, and disks that +// fill slowly until someone cleans up. + +// seed is 0 … 1, fixed for a host and what it's for. +func seed(name, what string) float64 { + h := fnv.New64a() + h.Write([]byte(name)) + h.Write([]byte{0}) + h.Write([]byte(what)) + return float64(h.Sum64()%1_000_003) / 1_000_003 +} + +// jitter is -1 … 1, fixed for a host and a moment (to the second), so a +// reading looks like a reading and the same moment always reads the same. +func jitter(name string, t time.Time) float64 { + h := fnv.New64a() + h.Write([]byte(name)) + var b [8]byte + u := uint64(t.Unix()) + for i := range b { + b[i] = byte(u >> (8 * i)) + } + h.Write(b[:]) + return float64(h.Sum64()%2_000_001)/1_000_000 - 1 +} + +// siteZone is the hours from UTC where a host's site is: the first site +// in Phoenix, then the US east coast, London and Frankfurt in turn. +func siteZone(name string) time.Duration { + n := fleetIndex(-1, name) + if n < 0 { + return -7 * time.Hour // the Windows servers are at the first site + } + return []time.Duration{-7 * time.Hour, -5 * time.Hour, 0, time.Hour}[(n/siteSize)%4] +} + +// workday is 0 … 1: how busy the working day is at t, where the host is. +// It peaks early afternoon, is near nothing at night, and a weekend is a +// third of a weekday. +func workday(name string, t time.Time) float64 { + local := t.Add(siteZone(name)).UTC() + h := float64(local.Hour()) + float64(local.Minute())/60 + d := 0.0 + if h > 7 && h < 20 { + d = math.Sin(math.Pi * (h - 7) / 13) + } + if wd := local.Weekday(); wd == time.Saturday || wd == time.Sunday { + d /= 3 + } + return d +} + +// cpuAt is a host's CPU use (percent) at t, without the moment's jitter. +// A third of hosts run a nightly job at 02:00 for half an hour. +func cpuAt(name string, t time.Time) float64 { + base := 3 + 17*seed(name, "cpu") + swing := base * (0.5 + 1.5*seed(name, "cpu swing")) + cpu := base + swing*workday(name, t) + if seed(name, "nightly") < 0.33 { + local := t.Add(siteZone(name)).UTC() + if local.Hour() == 2 && local.Minute() < 30 { + cpu += 25 + 30*seed(name, "nightly size") + } + } + return min(cpu, 97) +} + +// cpuNow is cpuAt with the moment's jitter, as one reading. +func cpuNow(name string, t time.Time) float64 { + return max(0.5, cpuAt(name, t)*(1+0.25*jitter(name, t))) +} + +// memAt is a host's memory use (percent) at t. Two hosts in five run +// something that grows until it's restarted, once a week. +func memAt(name string, t time.Time) float64 { + m := 30 + 40*seed(name, "mem") + 5*workday(name, t) + if seed(name, "leaks") < 0.4 { + week := 7 * 24 * time.Hour + phase := time.Duration(seed(name, "leak phase") * float64(week)) + frac := float64((t.Add(phase).UnixNano())%int64(week)) / float64(week) + m += 14 * frac + } + return min(m, 88) // memory warns at 90 +} + +// diskAt is how full (0 … 1) a host's disk k is at t: a level of its own, +// growing slowly over a cycle of one to three months until a clean-up +// brings it back. fleet-0023's log volume is the one that fills: from 58% +// to 86% in ten days, so it warns now and then. +func diskAt(name string, k int, t time.Time) float64 { + what := []string{"root", "data"}[min(k, 1)] + base := 0.12 + 0.48*seed(name, what) + grow := 0.04 + 0.12*seed(name, what+" growth") + cycle := time.Duration(30+60*seed(name, what+" cycle")) * 24 * time.Hour + if name == hostName(23) && k == 1 { + base, grow, cycle = 0.58, 0.28, 10*24*time.Hour + } + phase := time.Duration(seed(name, what+" phase") * float64(cycle)) + frac := float64(t.Add(phase).UnixNano()%int64(cycle)) / float64(cycle) + return min(base+grow*frac, 0.95) +} diff --git a/scripts/fleet/main.go b/scripts/fleet/main.go index 0e18ad9..4219e42 100644 --- a/scripts/fleet/main.go +++ b/scripts/fleet/main.go @@ -191,6 +191,9 @@ func run(args []string) { vaultDir := fs.String("vault", "", "serve two fake Vault clusters over HTTPS on loopback, and write their vault: list for obs-beacond.yaml to this directory's vault.yaml (\"\": none)") demo := fs.Bool("demo", false, "hosts report the demo's checks, units, backups and data layer, and fleet serves its Patroni cluster (fleet config -demo, same number of agents)") probe := fs.Bool("probe", true, "time how long a change in cluster-00 takes to show on every copy (needs dev_signin)") + histDB := fs.String("history", "", "first write the last days into this obs-beacond database (SQLite; history.go), unless it has some already (\"\": none)") + histKey := fs.String("state-key", "", "the database's state_key_file, for -history") + histDays := fs.Int("history-days", 30, "how many days -history writes") _ = fs.Parse(args) type agent struct{ host, token string } @@ -279,26 +282,83 @@ func run(args []string) { } var made, delivered, queued atomic.Int64 log.Printf("fleet: %d agents every %s, %d containers each, to %s, for %s", len(agents), *interval, *containers, *servers, *dur) + all := append(slices.Clone(agents), winAgents...) + hosts := make([]*host, len(all)) + for i, a := range all { + var h *host + switch { + case i >= len(agents): // a Windows server (windows.go) + h = newWindowsHost(a.host, *interval) + default: + h = newHost(i, a.host, *containers, *interval) + } + if *demo && h.win == nil { + h.demo = newDemoHost(fleetIndex(i, a.host), len(agents), demoAll) + h.demo.setup(&h.r) + } + if *withMap && h.win == nil { + h.m = fakeMap(i, a.host, h.r.Containers) + h.certs, h.files = hostCerts(i, a.host) + } + hosts[i] = h + } + // The clusters, made now so the history can be written from them. + type sim struct { + c *fakeCluster + probe bool + } + var sims []sim + for _, a := range clusterAgents { + sims = append(sims, sim{newCluster(a.host, a.token, shape), a.host == "cluster-00"}) + } + if *pveGuests > 0 && len(agents) > proxmoxHost { + a := agents[proxmoxHost] + sims = append(sims, sim{c: newProxmox(a.host, a.token, *pveGuests)}) + log.Printf("fleet: Proxmox cluster %s, %d nodes, %d guests, read by %s", proxmoxCluster, proxmoxNodes, *pveGuests, a.host) + } + if *vsphereVMs > 0 && len(agents) > vcenterHost { + a := agents[vcenterHost] + sims = append(sims, sim{c: newVSphere(a.host, a.token, *vsphereVMs)}) + log.Printf("fleet: vCenter %s, %d ESXi hosts, %d VMs, read by %s", vcenterName, esxiHosts, *vsphereVMs, a.host) + } + if *ntnxVMs > 0 && len(agents) > nutanixHost { + a := agents[nutanixHost] + sims = append(sims, sim{c: newNutanix(a.host, a.token, *ntnxVMs)}) + log.Printf("fleet: Prism Central %s, %d clusters of %d AHV hosts, %d VMs, read by %s", prismCentral, ntnxClusters, ahvPerPE, *ntnxVMs, a.host) + } + if n := min(*pureN, len(pureArrays)); n > 0 && len(agents) > pureHost { + a := agents[pureHost] + for k := range n { + sims = append(sims, sim{c: newPure(a.host, a.token, k)}) + } + log.Printf("fleet: %d Pure arrays (%v), read by %s", n, pureArrays[:n], a.host) + } + if n := min(*ibmiN, len(ibmiPartitions)); n > 0 && len(agents) > ibmiHost { + a := agents[ibmiHost] + for k := range n { + sims = append(sims, sim{c: newIBMi(a.host, a.token, k)}) + } + sims = append(sims, sim{c: newHMC(a.host, a.token)}) + log.Printf("fleet: %d IBM i partitions (%v) and the HMC %s, read by %s", n, ibmiPartitions[:n], hmcName, a.host) + } + if *histDB != "" { + past := pastFleet{hosts: hosts, vaults: *vaultDir != "", mail: *mailDir != ""} + for _, s := range sims { + past.clusters = append(past.clusters, s.c) + } + for _, c := range demoAll { + past.checks = append(past.checks, c.id) + } + if err := writeHistory(*histDB, *histKey, *histDays, past, time.Now()); err != nil { + log.Fatalf("fleet: history: %v", err) + } + } var wg sync.WaitGroup - for i, a := range append(slices.Clone(agents), winAgents...) { + for i, a := range all { wg.Add(1) go func() { defer wg.Done() - var h *host - switch { - case i >= len(agents): // a Windows server (windows.go) - h = newWindowsHost(a.host, *interval) - default: - h = newHost(i, a.host, *containers, *interval) - } - if *demo && h.win == nil { - h.demo = newDemoHost(fleetIndex(i, a.host), len(agents), demoAll) - h.demo.setup(&h.r) - } - if *withMap && h.win == nil { - h.m = fakeMap(i, a.host, h.r.Containers) - h.certs, h.files = hostCerts(i, a.host) - } + h := hosts[i] // Agents start at random points in the interval, as a fleet does. select { case <-ctx.Done(): @@ -362,36 +422,17 @@ func run(args []string) { log.Fatalf("fleet: %v", err) } } - for _, a := range clusterAgents { + for _, s := range sims { wg.Add(1) go func() { defer wg.Done() - c := newCluster(a.host, a.token, shape) var probed func(string, time.Time) - if cookies != nil && a.host == "cluster-00" { + if cookies != nil && s.probe { probed = func(word string, at time.Time) { watchProbe(ctx, client, urls, cookies, word, at, shown) } } - runCluster(ctx, c, client, pick, batches, probed) + runCluster(ctx, s.c, client, pick, batches, probed) }() } - if *pveGuests > 0 && len(agents) > proxmoxHost { - a := agents[proxmoxHost] - wg.Add(1) - go func() { - defer wg.Done() - runCluster(ctx, newProxmox(a.host, a.token, *pveGuests), client, pick, batches, nil) - }() - log.Printf("fleet: Proxmox cluster %s, %d nodes, %d guests, read by %s", proxmoxCluster, proxmoxNodes, *pveGuests, a.host) - } - if *vsphereVMs > 0 && len(agents) > vcenterHost { - a := agents[vcenterHost] - wg.Add(1) - go func() { - defer wg.Done() - runCluster(ctx, newVSphere(a.host, a.token, *vsphereVMs), client, pick, batches, nil) - }() - log.Printf("fleet: vCenter %s, %d ESXi hosts, %d VMs, read by %s", vcenterName, esxiHosts, *vsphereVMs, a.host) - } if *vaultDir != "" { serveVaults(ctx, *vaultDir) } @@ -407,42 +448,6 @@ func run(args []string) { if *akamaiDir != "" { serveAkamai(ctx, *akamaiDir) } - if *ntnxVMs > 0 && len(agents) > nutanixHost { - a := agents[nutanixHost] - wg.Add(1) - go func() { - defer wg.Done() - runCluster(ctx, newNutanix(a.host, a.token, *ntnxVMs), client, pick, batches, nil) - }() - log.Printf("fleet: Prism Central %s, %d clusters of %d AHV hosts, %d VMs, read by %s", prismCentral, ntnxClusters, ahvPerPE, *ntnxVMs, a.host) - } - if n := min(*pureN, len(pureArrays)); n > 0 && len(agents) > pureHost { - a := agents[pureHost] - for k := range n { - wg.Add(1) - go func() { - defer wg.Done() - runCluster(ctx, newPure(a.host, a.token, k), client, pick, batches, nil) - }() - } - log.Printf("fleet: %d Pure arrays (%v), read by %s", n, pureArrays[:n], a.host) - } - if n := min(*ibmiN, len(ibmiPartitions)); n > 0 && len(agents) > ibmiHost { - a := agents[ibmiHost] - for k := range n { - wg.Add(1) - go func() { - defer wg.Done() - runCluster(ctx, newIBMi(a.host, a.token, k), client, pick, batches, nil) - }() - } - wg.Add(1) - go func() { - defer wg.Done() - runCluster(ctx, newHMC(a.host, a.token), client, pick, batches, nil) - }() - log.Printf("fleet: %d IBM i partitions (%v) and the HMC %s, read by %s", n, ibmiPartitions[:n], hmcName, a.host) - } if *ui > 0 && len(agents) > 0 { wg.Add(1) go func() { @@ -726,13 +731,15 @@ func (h *host) next(now time.Time) report.Report { r := h.r r.Time = now.UTC() r.Uptime = now.Sub(h.start).Seconds() - r.CPU = 5 + 30*mrand.Float64() - r.Load = [3]float64{r.CPU / 25, r.CPU / 30, r.CPU / 35} - r.Mem.Available = uint64(float64(r.Mem.Total) * (0.3 + 0.4*mrand.Float64())) - r.Net = report.Net{RxBps: 1e5 * mrand.Float64(), TxBps: 5e4 * mrand.Float64()} + r.CPU = math.Round(cpuNow(r.Host, now)*10) / 10 // load.go + cores := float64(max(r.CPUs, 1)) + r.Load = [3]float64{cores * r.CPU / 100, cores * cpuAt(r.Host, now.Add(-5*time.Minute)) / 100, cores * cpuAt(r.Host, now.Add(-15*time.Minute)) / 100} + r.Mem.Available = uint64(float64(r.Mem.Total) * (1 - memAt(r.Host, now)/100)) + busy := 0.2 + workday(r.Host, now) + r.Net = report.Net{RxBps: 4e5 * busy * (1 + 0.3*jitter(r.Host, now)), TxBps: 1.5e5 * busy * (1 + 0.3*jitter(r.Host+" tx", now))} disks := slices.Clone(r.Disks) for i := range disks { - disks[i].Used = disks[i].Size / 3 + disks[i].Used = uint64(float64(disks[i].Size) * diskAt(r.Host, i, now)) disks[i].Avail = disks[i].Size - disks[i].Used } r.Disks = disks diff --git a/scripts/fleet/scenario.go b/scripts/fleet/scenario.go index a920679..30f810f 100644 --- a/scripts/fleet/scenario.go +++ b/scripts/fleet/scenario.go @@ -89,3 +89,45 @@ var ( spellFSCrash = spell{every: 8 * time.Hour, at: 7 * time.Hour, on: time.Hour} spellActivation = spell{every: 3 * time.Hour, at: 20 * time.Minute, on: 80 * time.Minute} ) + +// past is a problem as obs-beacond titles it, for the incidents the demo's +// history starts with (history.go): one for each time its spell came and +// went before the demo started. The keys and titles are obs-beacond's +// own, so the past and the present read alike. +type past struct { + s spell + key, sev, title, host string + href, detail, note string // note: what someone wrote afterwards, on the first of the stories' incidents in a day +} + +var pastProblems = []past{ + {s: spellLegacyDC, key: "akamai/property/legacy.corp.example.akadns.net", sev: "critical", href: "/akamai", + title: "Akamai GTM: legacy.corp.example.akadns.net has no live datacenter", + detail: "Every datacenter's server is failing the liveness test, so GTM answers with all of them, or none.", + note: "Ashburn lost its upstream link for half an hour. The legacy app has one datacenter, so GTM had nowhere else to send it."}, + {s: spellLegacyDC, key: "akamai/traffic/400019", sev: "critical", href: "/akamai", + title: "Akamai: 30% of CP code 400019's requests are failing with 5xx", + detail: "Edge errors for the legacy property: its origin is the Ashburn server GTM can't reach."}, + {s: spellLegacyDC, key: "akamai/server/legacy.corp.example.akadns.net/198.51.100.50", sev: "warning", href: "/akamai", + title: "198.51.100.50 in Ashburn is failing GTM's liveness test for legacy.corp.example.akadns.net"}, + {s: spellReplicaBehind, key: "cloudflare/errors/shop.example", sev: "warning", href: "/cloudflare", + title: "shop.example: 7% of requests failing", + detail: "Most on checkout.shop.example: 503s while HAProxy had one PostgreSQL replica for the shop's reads.", + note: "fleet-0032 fell 40 MB behind during the nightly vacuum and HAProxy took it out; the one replica left couldn't carry checkout's reads. Moving the vacuum out of business hours."}, + {s: spellAppExits, key: "container/fleet-0007/app-04", sev: "critical", host: "fleet-0007", href: "/containers", + title: "app-04 on fleet-0007: not running"}, + {s: spellDRPaths, key: "cluster/fleet-0040:fa-dr-01/connection/fa-prod-01", sev: "critical", host: "fleet-0040:fa-dr-01", href: "/clusters", + title: "Replication to fa-prod-01 in fa-dr-01 (fleet-0040): connected, but its paths aren't redundant"}, + {s: spellLBFlaps, key: "cloudflare/origin/site-1-web/fleet-0052", sev: "warning", host: "fleet-0052", href: "/cloudflare", + title: "fleet-0052 is failing site-1-web's health check"}, + {s: spellLBFlaps, key: "akamai/server/api.shop.example.akadns.net/203.0.2.11", sev: "warning", host: "fleet-0101", href: "/akamai", + title: "203.0.2.11 in Singapore is failing GTM's liveness test for api.shop.example.akadns.net"}, + {s: spellVaultEdgeDown, key: "node/obs-beacond:vault-edge/vault-edge-e3", sev: "critical", host: "obs-beacond:vault-edge", + href: "/workloads?source=obs-beacond%3Avault-edge&q=vault-edge-e3", title: "Server vault-edge-e3 in vault-edge (obs-beacond) is down"}, + {s: spellCrashloop, key: "workload/cluster-00/ns-07/deployment/app-097", sev: "warning", host: "cluster-00", + href: "/workloads?source=cluster-00&ns=ns-07&q=app-097", title: "deployment/app-097 in ns-07 (cluster-00): 1 of 10 pods: 1 restarting (CrashLoopBackOff)"}, + {s: spellLegacy502, key: "check/web-legacy", sev: "warning", href: "/checks", title: "legacy.example.com is down"}, + {s: spellBlocklisted, key: "check/rbl-mx2", sev: "warning", href: "/checks", title: "mx2.example.com on block lists is down"}, + {s: spellLateBackup, key: "backup/fleet-0050/received/fleet-0080/pg-dump", sev: "serious", host: "fleet-0050", href: "/backups", + title: "fleet-0080's pg-dump backup on fleet-0050 is 26 hours old"}, +} diff --git a/scripts/fleet/vault.go b/scripts/fleet/vault.go index 9c47ebd..ff40a12 100644 --- a/scripts/fleet/vault.go +++ b/scripts/fleet/vault.go @@ -74,9 +74,9 @@ func serveVaults(ctx context.Context, dir string) { now := time.Now() vaults := []*fakeVault{ newFakeVault("vault-prod", "1.21.4+ent", true, "pki_int", now, 300*24*time.Hour, 400, 2000, - []fakeRaft{{"v1", "leader", "alive"}, {"v2", "voter", "alive"}, {"v3", "voter", "alive"}}, 1, 18_000), + []fakeRaft{{"v1", "leader", "alive"}, {"v2", "voter", "alive"}, {"v3", "voter", "alive"}}, 1, vaultLeaseBase["vault-prod"]), newFakeVault("vault-edge", "2.4.1", false, "pki_edge", now, 12*24*time.Hour, 40, 0, - []fakeRaft{{"e1", "leader", "alive"}, {"e2", "voter", "alive"}, {"e3", "voter", "alive"}}, 1, 900), + []fakeRaft{{"e1", "leader", "alive"}, {"e2", "voter", "alive"}, {"e3", "voter", "alive"}}, 1, vaultLeaseBase["vault-edge"]), } var yaml strings.Builder yaml.WriteString("vault:\n") @@ -230,7 +230,7 @@ func (v *fakeVault) ServeHTTP(w http.ResponseWriter, r *http.Request) { case p == "auth/token/renew-self": out = map[string]any{"auth": map[string]any{"lease_duration": 24 * 3600}} case p == "sys/metrics": - l := v.leases + int(float64(v.leases)/10*math.Sin(float64(now.Unix())/3600)) + l := vaultLeases(v.leases, now) fmt.Fprintf(w, "# TYPE vault_expire_num_leases gauge\nvault_expire_num_leases %d\nvault_expire_num_irrevocable_leases 0\n", l) if v.enterprise { fmt.Fprint(w, "vault_audit_log_request_failure 0\nvault_audit_log_response_failure 0\n") @@ -268,3 +268,14 @@ func (v *fakeVault) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(out) } + +// vaultLeaseBase is each fake Vault's usual number of leases. +var vaultLeaseBase = map[string]int{"vault-prod": 18_000, "vault-edge": 900} + +// vaultLeases is how many leases a Vault holds at t: more in the working +// day, when people and jobs log in, with a little noise. +func vaultLeases(base int, t time.Time) int { + h := float64(t.UTC().Hour()) + float64(t.UTC().Minute())/60 + day := math.Sin(2 * math.Pi * (h - 14) / 24) // UTC 20:00 is early afternoon in Phoenix + return base + int(float64(base)*(0.12*day+0.02*jitter("leases", t.Truncate(time.Minute)))) +}