Demo: start with 30 days behind it #51

Merged
jcoffey-dev merged 2 commits from feature/demo-history into main 2026-11-16 06:11:27 +00:00
9 changed files with 709 additions and 82 deletions

No files matched your search

+16
View File
@@ -46,6 +46,11 @@ obs-beacond and the simulator, so nothing in it can reach anything else:
- `fleet config -demo` writes the server's pieces; `fleet run -demo` runs
the fleet (add `-ibmi 4` for IBM i and the HMC). Start the fleet first so
its stand-ins are up for obs-beacond's first readings.
- `fleet run -history <database> -state-key <obs-beacond's state key>`
first writes the last 30 days into obs-beacond's (SQLite) database, so
the graphs, reports and incident history aren't empty on a fresh demo
(below). It takes under a minute, before the stand-ins start, so wait
for them that long; it leaves a database that already has history alone.
- The root is read-only and the state is in tmpfs, so restarting the
container is a fresh demo; restart it on a timer (weekly is plenty) so
history doesn't pile up.
@@ -59,3 +64,14 @@ problems that come and go through the day, so incidents open, resolve and
show up under Recently resolved. Because the plan runs on the clock, two
visitors an hour apart see different problems, and a restarted demo shows
the same ones the old one would have.
The demo starts with a month behind it, worked out from the same plan and
clock (`scripts/fleet/history.go`): every host's hourly CPU, memory and
disks (a working day in its site's time zone, quieter weekends, memory
that creeps until a weekly restart, disks that fill until a clean-up), the
clusters' nodes (and a week of their controllers and VMs, to keep the
database small), the arrays' I/O, the IBM i partitions' CPU, the
Vaults' leases, the mail monitor's rounds, availability for the reports,
and an incident for each time a problem came and went, with notes on the
stories saying what the cause was. Its incident numbers carry on from
there.
+9
View File
@@ -82,6 +82,15 @@ restarted demo carries on where the clock is. Incidents open and resolve
through the day, and a dozen or two are open at any moment. The mail
server is healthy, to show what normal looks like.
What a host uses is a function of its name and the clock
(`scripts/fleet/load.go`), not a dice roll, so `-history <database>
-state-key <file>` can write the last 30 days (`-history-days`) before
anything reports (`history.go`): hosts' and workloads' hours, mail rounds,
statuses, and an incident for each past spell of the problems in
`pastProblems` (scenario.go), whose keys and titles are obs-beacond's
own. When you change what a spell's problem says in obs-beacond, change
its line there too.
## Dependencies and notices
THIRD_PARTY_NOTICES.md lists every third-party component inside the
+50
View File
@@ -245,3 +245,53 @@ func (d *DB) Prune(before time.Time) error {
}
return nil
}
// PutHostHours is PutHostHour for many, in one transaction: a backfill
// (the demo's 30 days, scripts/fleet) or an import.
func (d *DB) PutHostHours(hs []HostHour) error {
t, err := d.begin()
if err != nil {
return err
}
defer t.Rollback()
for _, h := range hs {
b, _ := json.Marshal(h.Disks)
if _, err := t.Exec(`INSERT INTO host_hours (host, hour, cpu, cpu_max, mem, mem_max, disks) VALUES (?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (host, hour) DO UPDATE SET cpu=excluded.cpu, cpu_max=excluded.cpu_max, mem=excluded.mem, mem_max=excluded.mem_max, disks=excluded.disks`,
h.Host, h.Hour.Unix(), h.CPU, h.CPUMax, h.Mem, h.MemMax, string(b)); err != nil {
return err
}
}
return t.Commit()
}
// PutMailRounds is PutMailRound for many, in one transaction.
func (d *DB) PutMailRounds(ms []MailRound) error {
t, err := d.begin()
if err != nil {
return err
}
defer t.Rollback()
for _, m := range ms {
if _, err := t.Exec(`INSERT INTO mail_rounds (at, seconds, queue) VALUES (?, ?, ?)
ON CONFLICT (at) DO UPDATE SET seconds=excluded.seconds, queue=excluded.queue`, m.At.Unix(), m.Seconds, m.Queue); err != nil {
return err
}
}
return t.Commit()
}
// LogStatuses is LogStatus for many, in one transaction.
func (d *DB) LogStatuses(cs []Change) error {
t, err := d.begin()
if err != nil {
return err
}
defer t.Rollback()
for _, c := range cs {
if _, err := t.Exec(`INSERT INTO status_log (entity, at, status) VALUES (?, ?, ?)`, c.Entity, c.At.Unix(), c.Status); err != nil {
return err
}
}
return t.Commit()
}
+292
View File
@@ -0,0 +1,292 @@
package main
import (
"fmt"
"log"
"math"
"slices"
"time"
"github.com/cairnobs/cairnobs/internal/report"
"github.com/cairnobs/cairnobs/internal/state"
)
// The demo's past (#39): before anything reports, fleet run -history
// writes the last 30 days into obs-beacond's database, worked out from the
// same clock the live demo runs on (load.go, scenario.go), so the graphs
// have their days and the Alerts page and reports have their incidents:
//
// - each host's hours: CPU and memory averages and peaks, disks;
// - each cluster's workloads' hours, an array's I/O and space, an IBM i
// partition's CPU, a Vault's leases;
// - the mail monitor's rounds;
// - hosts' and checks' statuses, from the start;
// - an incident each time a past problem (pastProblems) came and went,
// with a note from whoever looked at the stories.
// pastFlaps are the statuses that change with a spell, for the reports'
// availability: an entity takes status while it's on, and after when
// it's over.
var pastFlaps = []struct {
entity string
s spell
status, after string
}{
{"check/web-legacy", spellLegacy502, "warning", "good"},
{"check/rbl-mx2", spellBlocklisted, "warning", "good"},
{"svc/sites", spellLegacy502, "warning", "good"},
{"svc/mail", spellBlocklisted, "warning", "good"},
{"svc/postgres", spellReplicaBehind, "warning", "good"},
{"svc/haproxy", spellReplicaBehind, "warning", "good"},
// One copy of something there are several of is the service degraded,
// not down (topology.go, redundant parts).
{"svc/apps", spellAppExits, "warning", "good"},
{"svc/redis", spellRedisFails, "warning", "good"},
}
// pastFleet is what the history is written for.
type pastFleet struct {
hosts []*host
clusters []*fakeCluster
vaults bool // the Vault stand-ins run
mail bool // the mail stand-in runs
checks []string
}
func writeHistory(path, keyFile string, days int, f pastFleet, now time.Time) error {
start := time.Now()
db, err := state.Open(path, keyFile)
if err != nil {
return err
}
defer db.Close()
if !db.HistoryStart().IsZero() {
log.Printf("fleet: %s already has history; left as it is", path)
return nil
}
from := now.Truncate(time.Hour).Add(-time.Duration(days) * 24 * time.Hour)
to := now.Truncate(time.Hour) // the hour now is obs-beacond's to write
var hs []state.HostHour
var statuses []state.Change
for _, h := range f.hosts {
name := h.r.Host
statuses = append(statuses, state.Change{Entity: "host/" + name, At: from, Status: "good"})
for t := from; t.Before(to); t = t.Add(time.Hour) {
hs = append(hs, hostHour(h, t))
}
}
for _, id := range f.checks {
statuses = append(statuses, state.Change{Entity: "check/" + id, At: from, Status: "good"})
}
if f.checks != nil { // the demo's services (demoTopology)
for _, id := range []string{"shop", "api", "sites", "mail", "apps", "haproxy", "postgres", "redis", "nats"} {
statuses = append(statuses, state.Change{Entity: "svc/" + id, At: from, Status: "good"})
}
// Flexible SSL on legacy.example is a lasting warning, and the
// shop is behind Cloudflare.
statuses = append(statuses, state.Change{Entity: "svc/cloudflare", At: from, Status: "warning"},
state.Change{Entity: "svc/shop", At: from.Add(time.Second), Status: "warning"})
for _, fl := range pastFlaps {
s := fl.s
for b := s.began(from).Add(s.every); b.Add(s.on).Before(now); b = b.Add(s.every) {
statuses = append(statuses, state.Change{Entity: fl.entity, At: b.Add(20 * time.Second), Status: fl.status},
state.Change{Entity: fl.entity, At: b.Add(s.on + 20*time.Second), Status: fl.after})
}
}
slices.SortStableFunc(statuses, func(a, b state.Change) int { return a.At.Compare(b.At) })
}
if err := db.PutHostHours(hs); err != nil {
return fmt.Errorf("host hours: %w", err)
}
if err := db.LogStatuses(statuses); err != nil {
return fmt.Errorf("statuses: %w", err)
}
nw := 0
// Thousands of controllers and VMs get a week: a month of each is most
// of the database, and the demo's state is a 256 MB tmpfs.
few := to.Add(-time.Duration(min(days, 7)) * 24 * time.Hour)
for _, c := range f.clusters {
ws := clusterHours(c, from, few, to)
if err := db.PutWorkloadHours(ws); err != nil {
return fmt.Errorf("%s: %w", c.name, err)
}
nw += len(ws)
}
if f.vaults {
var ws []state.WorkloadHour
for name, base := range vaultLeaseBase {
for t := from; t.Before(to); t = t.Add(time.Hour) {
avg, top := 0.0, 0.0
for m := range 4 {
l := float64(vaultLeases(base, t.Add(time.Duration(m)*15*time.Minute)))
avg += l / 4
top = max(top, l)
}
ws = append(ws, state.WorkloadHour{Source: report.SourceKey("obs-beacond", name), Workload: name, Hour: t, Kind: report.KindCluster,
Leases: math.Round(avg), LeasesMax: top})
}
}
if err := db.PutWorkloadHours(ws); err != nil {
return fmt.Errorf("vault: %w", err)
}
nw += len(ws)
}
var ms []state.MailRound
if f.mail {
for t := from; t.Before(now.Add(-time.Minute)); t = t.Add(time.Minute) {
k := int(t.Unix() / 60)
ms = append(ms, state.MailRound{At: t, Seconds: 1.5 + float64(k*7919%4500)/1000, Queue: 3 + k%6})
}
if err := db.PutMailRounds(ms); err != nil {
return fmt.Errorf("mail rounds: %w", err)
}
}
n, err := writePastIncidents(db, from, now)
if err != nil {
return fmt.Errorf("incidents: %w", err)
}
log.Printf("fleet: wrote %d days of history in %s: %d host hours, %d workload hours, %d mail rounds, %d incidents",
days, time.Since(start).Round(time.Millisecond), len(hs), nw, len(ms), n)
return nil
}
// hostHour is one host's hour, from a reading every five minutes, as
// obs-beacond's own (from every report) would roughly be.
func hostHour(h *host, t time.Time) state.HostHour {
name := h.r.Host
out := state.HostHour{Host: name, Hour: t}
const n = 12
for k := range n {
at := t.Add(time.Duration(k)*5*time.Minute + 2*time.Minute)
c, m := cpuNow(name, at), memAt(name, at)
out.CPU += c / n
out.Mem += m / n
out.CPUMax = max(out.CPUMax, c*(1.1+0.3*seed(name, at.Format("15"))))
out.MemMax = max(out.MemMax, m+1.5)
}
out.CPU, out.CPUMax = round1(out.CPU), round1(min(out.CPUMax, 100))
out.Mem, out.MemMax = round1(out.Mem), round1(min(out.MemMax, 100))
end := t.Add(time.Hour - time.Minute)
for i, d := range h.r.Disks {
out.Disks = append(out.Disks, state.DiskSample{Mount: d.Mount, Used: uint64(float64(d.Size) * diskAt(name, i, end)), Total: d.Size})
}
return out
}
func round1(f float64) float64 { return math.Round(f*10) / 10 }
// clusterHours is a cluster's workloads' hours, from two readings an hour,
// keyed the way obs-beacond keeps them (its historyKey): from from for
// the cluster itself and its nodes, from few for the rest.
func clusterHours(c *fakeCluster, from, few, to time.Time) []state.WorkloadHour {
if c.noUsage && c.usageFn == nil {
return nil
}
src := report.SourceKey(c.name, c.source)
c.mu.Lock()
names := map[string]report.Workload{}
for id, r := range c.recs {
names[id] = report.Workload{Namespace: r.Namespace, Name: r.Name, Kind: r.Kind}
}
c.mu.Unlock()
var out []state.WorkloadHour
for t := from; t.Before(to); t = t.Add(time.Hour) {
sum := map[string]*state.WorkloadHour{}
const n = 2
for k := range n {
at := t.Add(time.Duration(k)*30*time.Minute + 10*time.Minute)
us := c.usageFn
if us == nil {
us = c.usage
}
for _, u := range us(at) {
r, ok := names[u.ID]
if !ok || t.Before(few) && r.Kind != report.KindNode && r.Kind != report.KindCluster {
continue
}
key := r.Name
if r.Namespace != "" {
key = r.Namespace + "/" + r.Name
}
h := sum[key]
if h == nil {
h = &state.WorkloadHour{Source: src, Workload: key, Hour: t, Kind: r.Kind}
sum[key] = h
}
h.CPU += u.CPU / n
h.Mem += float64(u.Mem) / n
h.CPUMax, h.MemMax = max(h.CPUMax, u.CPU), max(h.MemMax, float64(u.Mem))
h.CPUOf, h.MemOf = u.CPUOf, float64(u.MemOf)
if io := u.IO; io != nil {
h.ReadOps += io.ReadOps / n
h.WriteOps += io.WriteOps / n
h.ReadUs += io.ReadUs / n
h.WriteUs += io.WriteUs / n
h.ReadBps += io.ReadBps / n
h.WriteBps += io.WriteBps / n
h.ReadUsMax, h.WriteUsMax = max(h.ReadUsMax, io.ReadUs*1.4), max(h.WriteUsMax, io.WriteUs*1.4)
h.Used, h.UsedOf = float64(io.Used), float64(io.UsedOf)
}
}
}
for _, h := range sum {
out = append(out, *h)
}
}
return out
}
// writePastIncidents writes an incident each time a past problem came and
// went in [from, now), numbered in order from B-0001, and tells
// obs-beacond where the numbers carry on.
func writePastIncidents(db *state.DB, from, now time.Time) (int, error) {
type occ struct {
p past
began time.Time
first bool // the first of its kind on its day: it gets the note
}
var all []occ
for _, p := range pastProblems {
if p.s.every == 0 {
continue
}
day := ""
for b := p.s.began(from).Add(p.s.every); b.Add(p.s.on).Before(now); b = b.Add(p.s.every) {
d := b.UTC().Format("2006-01-02")
all = append(all, occ{p, b, d != day})
day = d
}
}
slices.SortStableFunc(all, func(a, b occ) int { return a.began.Compare(b.began) })
for k, o := range all {
p := o.p
// Seen within a reading or two, told after alerts.for (2m), and
// resolved within a minute of its end.
opened := o.began.Add(time.Duration(10+seedInt(p.key, o.began, 40)) * time.Second)
fired := opened.Add(2 * time.Minute)
resolved := o.began.Add(p.s.on + time.Duration(15+seedInt(p.key+" end", o.began, 45))*time.Second)
inc := state.Incident{ID: fmt.Sprintf("B-%04d", k+1), Key: p.key, Severity: p.sev, Title: p.title, Detail: p.detail,
Host: p.host, Href: p.href, Opened: opened, Fired: fired, Notified: fired, Resolved: resolved}
if p.sev == "critical" && p.s.on > 10*time.Minute {
inc.AckBy, inc.AckAt = "On call", fired.Add(time.Duration(2+seedInt(p.key+" ack", o.began, 6))*time.Minute)
}
if err := db.PutIncident(inc); err != nil {
return 0, err
}
if p.note != "" && o.first {
if err := db.SetNote(inc.ID, state.Note{Text: p.note, By: "On call", At: resolved.Add(20 * time.Minute)}); err != nil {
return 0, err
}
}
}
return len(all), db.SetMeta("incident_seq", fmt.Sprint(len(all)))
}
// seedInt is 0 … n-1, fixed for a key and a moment.
func seedInt(key string, t time.Time, n int) int {
return int((jitter(key, t) + 1) / 2 * float64(n-1))
}
+88
View File
@@ -0,0 +1,88 @@
package main
import (
"encoding/base64"
"fmt"
"os"
"path/filepath"
"testing"
"time"
"github.com/cairnobs/cairnobs/internal/state"
)
// Hosts stay under the warnings (disks 80%, memory 90%), but fleet-0023's
// log volume, which crosses 80% for a while every ten days (#40).
func TestLoadStaysCalm(t *testing.T) {
now := time.Date(2026, 11, 16, 5, 0, 0, 0, time.UTC)
over := 0
for i := range 150 {
name := hostName(i)
for h := range 30 * 24 {
at := now.Add(-time.Duration(h) * time.Hour)
if c := cpuNow(name, at); c <= 0 || c > 100 {
t.Fatalf("%s at %s: CPU %.1f", name, at, c)
}
if m := memAt(name, at); m >= 90 {
t.Fatalf("%s at %s: memory %.1f", name, at, m)
}
for k := range 2 {
if d := diskAt(name, k, at); d >= 0.8 {
if i != 23 || k != 1 {
t.Fatalf("%s disk %d at %s: %.2f", name, k, at, d)
}
over++
}
}
}
}
if over == 0 || over > 30*24/3 {
t.Errorf("fleet-0023's log volume is over 80%% for %d of 720 hours", over)
}
// Not all alike, as every disk at 33% was.
if diskAt(hostName(1), 0, now) == diskAt(hostName(2), 0, now) {
t.Error("two hosts' disks are the same")
}
}
func TestWriteHistory(t *testing.T) {
dir := t.TempDir()
key := filepath.Join(dir, "state.key")
if err := os.WriteFile(key, []byte(base64.StdEncoding.EncodeToString(make([]byte, 32))), 0o600); err != nil {
t.Fatal(err)
}
db := filepath.Join(dir, "beacon.db")
hosts := []*host{newHost(1, hostName(1), 2, 10*time.Second), newHost(2, hostName(2), 2, 10*time.Second)}
now := time.Now()
if err := writeHistory(db, key, 3, pastFleet{hosts: hosts, vaults: true, mail: true, checks: []string{"web-legacy"}}, now); err != nil {
t.Fatal(err)
}
d, err := state.Open(db, key)
if err != nil {
t.Fatal(err)
}
defer d.Close()
hs, _ := d.HostHours(now.Add(-4*24*time.Hour), now)
if len(hs) != 2*3*24 {
t.Errorf("%d host hours", len(hs))
}
incs, _ := d.IncidentsBetween(now.Add(-4*24*time.Hour), now)
if len(incs) < 20 || d.Meta("incident_seq") != fmt.Sprint(len(incs)) {
t.Errorf("%d incidents, incident_seq %s", len(incs), d.Meta("incident_seq"))
}
for _, i := range incs {
if i.Resolved.After(now) || !i.Resolved.After(i.Opened) {
t.Fatalf("%s %s: opened %s, resolved %s", i.ID, i.Key, i.Opened, i.Resolved)
}
}
if d.HistoryStart().IsZero() {
t.Error("no statuses")
}
// A second run leaves it alone.
if err := writeHistory(db, key, 3, pastFleet{hosts: hosts}, now); err != nil {
t.Fatal(err)
}
if hs2, _ := d.HostHours(now.Add(-4*24*time.Hour), now); len(hs2) != len(hs) {
t.Errorf("written twice: %d hours", len(hs2))
}
}
+112
View File
@@ -0,0 +1,112 @@
package main
import (
"hash/fnv"
"math"
"time"
)
// What a fake host uses, worked out from its name and the clock alone, so
// the live reports and the 30 days written before them (history.go)
// agree: a working day in its site's time zone and quieter weekends,
// memory that creeps on some hosts until a weekly restart, and disks that
// fill slowly until someone cleans up.
// seed is 0 … 1, fixed for a host and what it's for.
func seed(name, what string) float64 {
h := fnv.New64a()
h.Write([]byte(name))
h.Write([]byte{0})
h.Write([]byte(what))
return float64(h.Sum64()%1_000_003) / 1_000_003
}
// jitter is -1 … 1, fixed for a host and a moment (to the second), so a
// reading looks like a reading and the same moment always reads the same.
func jitter(name string, t time.Time) float64 {
h := fnv.New64a()
h.Write([]byte(name))
var b [8]byte
u := uint64(t.Unix())
for i := range b {
b[i] = byte(u >> (8 * i))
}
h.Write(b[:])
return float64(h.Sum64()%2_000_001)/1_000_000 - 1
}
// siteZone is the hours from UTC where a host's site is: the first site
// in Phoenix, then the US east coast, London and Frankfurt in turn.
func siteZone(name string) time.Duration {
n := fleetIndex(-1, name)
if n < 0 {
return -7 * time.Hour // the Windows servers are at the first site
}
return []time.Duration{-7 * time.Hour, -5 * time.Hour, 0, time.Hour}[(n/siteSize)%4]
}
// workday is 0 … 1: how busy the working day is at t, where the host is.
// It peaks early afternoon, is near nothing at night, and a weekend is a
// third of a weekday.
func workday(name string, t time.Time) float64 {
local := t.Add(siteZone(name)).UTC()
h := float64(local.Hour()) + float64(local.Minute())/60
d := 0.0
if h > 7 && h < 20 {
d = math.Sin(math.Pi * (h - 7) / 13)
}
if wd := local.Weekday(); wd == time.Saturday || wd == time.Sunday {
d /= 3
}
return d
}
// cpuAt is a host's CPU use (percent) at t, without the moment's jitter.
// A third of hosts run a nightly job at 02:00 for half an hour.
func cpuAt(name string, t time.Time) float64 {
base := 3 + 17*seed(name, "cpu")
swing := base * (0.5 + 1.5*seed(name, "cpu swing"))
cpu := base + swing*workday(name, t)
if seed(name, "nightly") < 0.33 {
local := t.Add(siteZone(name)).UTC()
if local.Hour() == 2 && local.Minute() < 30 {
cpu += 25 + 30*seed(name, "nightly size")
}
}
return min(cpu, 97)
}
// cpuNow is cpuAt with the moment's jitter, as one reading.
func cpuNow(name string, t time.Time) float64 {
return max(0.5, cpuAt(name, t)*(1+0.25*jitter(name, t)))
}
// memAt is a host's memory use (percent) at t. Two hosts in five run
// something that grows until it's restarted, once a week.
func memAt(name string, t time.Time) float64 {
m := 30 + 40*seed(name, "mem") + 5*workday(name, t)
if seed(name, "leaks") < 0.4 {
week := 7 * 24 * time.Hour
phase := time.Duration(seed(name, "leak phase") * float64(week))
frac := float64((t.Add(phase).UnixNano())%int64(week)) / float64(week)
m += 14 * frac
}
return min(m, 88) // memory warns at 90
}
// diskAt is how full (0 … 1) a host's disk k is at t: a level of its own,
// growing slowly over a cycle of one to three months until a clean-up
// brings it back. fleet-0023's log volume is the one that fills: from 58%
// to 86% in ten days, so it warns now and then.
func diskAt(name string, k int, t time.Time) float64 {
what := []string{"root", "data"}[min(k, 1)]
base := 0.12 + 0.48*seed(name, what)
grow := 0.04 + 0.12*seed(name, what+" growth")
cycle := time.Duration(30+60*seed(name, what+" cycle")) * 24 * time.Hour
if name == hostName(23) && k == 1 {
base, grow, cycle = 0.58, 0.28, 10*24*time.Hour
}
phase := time.Duration(seed(name, what+" phase") * float64(cycle))
frac := float64(t.Add(phase).UnixNano()%int64(cycle)) / float64(cycle)
return min(base+grow*frac, 0.95)
}
+86 -79
View File
@@ -191,6 +191,9 @@ func run(args []string) {
vaultDir := fs.String("vault", "", "serve two fake Vault clusters over HTTPS on loopback, and write their vault: list for obs-beacond.yaml to this directory's vault.yaml (\"\": none)")
demo := fs.Bool("demo", false, "hosts report the demo's checks, units, backups and data layer, and fleet serves its Patroni cluster (fleet config -demo, same number of agents)")
probe := fs.Bool("probe", true, "time how long a change in cluster-00 takes to show on every copy (needs dev_signin)")
histDB := fs.String("history", "", "first write the last days into this obs-beacond database (SQLite; history.go), unless it has some already (\"\": none)")
histKey := fs.String("state-key", "", "the database's state_key_file, for -history")
histDays := fs.Int("history-days", 30, "how many days -history writes")
_ = fs.Parse(args)
type agent struct{ host, token string }
@@ -279,26 +282,83 @@ func run(args []string) {
}
var made, delivered, queued atomic.Int64
log.Printf("fleet: %d agents every %s, %d containers each, to %s, for %s", len(agents), *interval, *containers, *servers, *dur)
all := append(slices.Clone(agents), winAgents...)
hosts := make([]*host, len(all))
for i, a := range all {
var h *host
switch {
case i >= len(agents): // a Windows server (windows.go)
h = newWindowsHost(a.host, *interval)
default:
h = newHost(i, a.host, *containers, *interval)
}
if *demo && h.win == nil {
h.demo = newDemoHost(fleetIndex(i, a.host), len(agents), demoAll)
h.demo.setup(&h.r)
}
if *withMap && h.win == nil {
h.m = fakeMap(i, a.host, h.r.Containers)
h.certs, h.files = hostCerts(i, a.host)
}
hosts[i] = h
}
// The clusters, made now so the history can be written from them.
type sim struct {
c *fakeCluster
probe bool
}
var sims []sim
for _, a := range clusterAgents {
sims = append(sims, sim{newCluster(a.host, a.token, shape), a.host == "cluster-00"})
}
if *pveGuests > 0 && len(agents) > proxmoxHost {
a := agents[proxmoxHost]
sims = append(sims, sim{c: newProxmox(a.host, a.token, *pveGuests)})
log.Printf("fleet: Proxmox cluster %s, %d nodes, %d guests, read by %s", proxmoxCluster, proxmoxNodes, *pveGuests, a.host)
}
if *vsphereVMs > 0 && len(agents) > vcenterHost {
a := agents[vcenterHost]
sims = append(sims, sim{c: newVSphere(a.host, a.token, *vsphereVMs)})
log.Printf("fleet: vCenter %s, %d ESXi hosts, %d VMs, read by %s", vcenterName, esxiHosts, *vsphereVMs, a.host)
}
if *ntnxVMs > 0 && len(agents) > nutanixHost {
a := agents[nutanixHost]
sims = append(sims, sim{c: newNutanix(a.host, a.token, *ntnxVMs)})
log.Printf("fleet: Prism Central %s, %d clusters of %d AHV hosts, %d VMs, read by %s", prismCentral, ntnxClusters, ahvPerPE, *ntnxVMs, a.host)
}
if n := min(*pureN, len(pureArrays)); n > 0 && len(agents) > pureHost {
a := agents[pureHost]
for k := range n {
sims = append(sims, sim{c: newPure(a.host, a.token, k)})
}
log.Printf("fleet: %d Pure arrays (%v), read by %s", n, pureArrays[:n], a.host)
}
if n := min(*ibmiN, len(ibmiPartitions)); n > 0 && len(agents) > ibmiHost {
a := agents[ibmiHost]
for k := range n {
sims = append(sims, sim{c: newIBMi(a.host, a.token, k)})
}
sims = append(sims, sim{c: newHMC(a.host, a.token)})
log.Printf("fleet: %d IBM i partitions (%v) and the HMC %s, read by %s", n, ibmiPartitions[:n], hmcName, a.host)
}
if *histDB != "" {
past := pastFleet{hosts: hosts, vaults: *vaultDir != "", mail: *mailDir != ""}
for _, s := range sims {
past.clusters = append(past.clusters, s.c)
}
for _, c := range demoAll {
past.checks = append(past.checks, c.id)
}
if err := writeHistory(*histDB, *histKey, *histDays, past, time.Now()); err != nil {
log.Fatalf("fleet: history: %v", err)
}
}
var wg sync.WaitGroup
for i, a := range append(slices.Clone(agents), winAgents...) {
for i, a := range all {
wg.Add(1)
go func() {
defer wg.Done()
var h *host
switch {
case i >= len(agents): // a Windows server (windows.go)
h = newWindowsHost(a.host, *interval)
default:
h = newHost(i, a.host, *containers, *interval)
}
if *demo && h.win == nil {
h.demo = newDemoHost(fleetIndex(i, a.host), len(agents), demoAll)
h.demo.setup(&h.r)
}
if *withMap && h.win == nil {
h.m = fakeMap(i, a.host, h.r.Containers)
h.certs, h.files = hostCerts(i, a.host)
}
h := hosts[i]
// Agents start at random points in the interval, as a fleet does.
select {
case <-ctx.Done():
@@ -362,36 +422,17 @@ func run(args []string) {
log.Fatalf("fleet: %v", err)
}
}
for _, a := range clusterAgents {
for _, s := range sims {
wg.Add(1)
go func() {
defer wg.Done()
c := newCluster(a.host, a.token, shape)
var probed func(string, time.Time)
if cookies != nil && a.host == "cluster-00" {
if cookies != nil && s.probe {
probed = func(word string, at time.Time) { watchProbe(ctx, client, urls, cookies, word, at, shown) }
}
runCluster(ctx, c, client, pick, batches, probed)
runCluster(ctx, s.c, client, pick, batches, probed)
}()
}
if *pveGuests > 0 && len(agents) > proxmoxHost {
a := agents[proxmoxHost]
wg.Add(1)
go func() {
defer wg.Done()
runCluster(ctx, newProxmox(a.host, a.token, *pveGuests), client, pick, batches, nil)
}()
log.Printf("fleet: Proxmox cluster %s, %d nodes, %d guests, read by %s", proxmoxCluster, proxmoxNodes, *pveGuests, a.host)
}
if *vsphereVMs > 0 && len(agents) > vcenterHost {
a := agents[vcenterHost]
wg.Add(1)
go func() {
defer wg.Done()
runCluster(ctx, newVSphere(a.host, a.token, *vsphereVMs), client, pick, batches, nil)
}()
log.Printf("fleet: vCenter %s, %d ESXi hosts, %d VMs, read by %s", vcenterName, esxiHosts, *vsphereVMs, a.host)
}
if *vaultDir != "" {
serveVaults(ctx, *vaultDir)
}
@@ -407,42 +448,6 @@ func run(args []string) {
if *akamaiDir != "" {
serveAkamai(ctx, *akamaiDir)
}
if *ntnxVMs > 0 && len(agents) > nutanixHost {
a := agents[nutanixHost]
wg.Add(1)
go func() {
defer wg.Done()
runCluster(ctx, newNutanix(a.host, a.token, *ntnxVMs), client, pick, batches, nil)
}()
log.Printf("fleet: Prism Central %s, %d clusters of %d AHV hosts, %d VMs, read by %s", prismCentral, ntnxClusters, ahvPerPE, *ntnxVMs, a.host)
}
if n := min(*pureN, len(pureArrays)); n > 0 && len(agents) > pureHost {
a := agents[pureHost]
for k := range n {
wg.Add(1)
go func() {
defer wg.Done()
runCluster(ctx, newPure(a.host, a.token, k), client, pick, batches, nil)
}()
}
log.Printf("fleet: %d Pure arrays (%v), read by %s", n, pureArrays[:n], a.host)
}
if n := min(*ibmiN, len(ibmiPartitions)); n > 0 && len(agents) > ibmiHost {
a := agents[ibmiHost]
for k := range n {
wg.Add(1)
go func() {
defer wg.Done()
runCluster(ctx, newIBMi(a.host, a.token, k), client, pick, batches, nil)
}()
}
wg.Add(1)
go func() {
defer wg.Done()
runCluster(ctx, newHMC(a.host, a.token), client, pick, batches, nil)
}()
log.Printf("fleet: %d IBM i partitions (%v) and the HMC %s, read by %s", n, ibmiPartitions[:n], hmcName, a.host)
}
if *ui > 0 && len(agents) > 0 {
wg.Add(1)
go func() {
@@ -726,13 +731,15 @@ func (h *host) next(now time.Time) report.Report {
r := h.r
r.Time = now.UTC()
r.Uptime = now.Sub(h.start).Seconds()
r.CPU = 5 + 30*mrand.Float64()
r.Load = [3]float64{r.CPU / 25, r.CPU / 30, r.CPU / 35}
r.Mem.Available = uint64(float64(r.Mem.Total) * (0.3 + 0.4*mrand.Float64()))
r.Net = report.Net{RxBps: 1e5 * mrand.Float64(), TxBps: 5e4 * mrand.Float64()}
r.CPU = math.Round(cpuNow(r.Host, now)*10) / 10 // load.go
cores := float64(max(r.CPUs, 1))
r.Load = [3]float64{cores * r.CPU / 100, cores * cpuAt(r.Host, now.Add(-5*time.Minute)) / 100, cores * cpuAt(r.Host, now.Add(-15*time.Minute)) / 100}
r.Mem.Available = uint64(float64(r.Mem.Total) * (1 - memAt(r.Host, now)/100))
busy := 0.2 + workday(r.Host, now)
r.Net = report.Net{RxBps: 4e5 * busy * (1 + 0.3*jitter(r.Host, now)), TxBps: 1.5e5 * busy * (1 + 0.3*jitter(r.Host+" tx", now))}
disks := slices.Clone(r.Disks)
for i := range disks {
disks[i].Used = disks[i].Size / 3
disks[i].Used = uint64(float64(disks[i].Size) * diskAt(r.Host, i, now))
disks[i].Avail = disks[i].Size - disks[i].Used
}
r.Disks = disks
+42
View File
@@ -89,3 +89,45 @@ var (
spellFSCrash = spell{every: 8 * time.Hour, at: 7 * time.Hour, on: time.Hour}
spellActivation = spell{every: 3 * time.Hour, at: 20 * time.Minute, on: 80 * time.Minute}
)
// past is a problem as obs-beacond titles it, for the incidents the demo's
// history starts with (history.go): one for each time its spell came and
// went before the demo started. The keys and titles are obs-beacond's
// own, so the past and the present read alike.
type past struct {
s spell
key, sev, title, host string
href, detail, note string // note: what someone wrote afterwards, on the first of the stories' incidents in a day
}
var pastProblems = []past{
{s: spellLegacyDC, key: "akamai/property/legacy.corp.example.akadns.net", sev: "critical", href: "/akamai",
title: "Akamai GTM: legacy.corp.example.akadns.net has no live datacenter",
detail: "Every datacenter's server is failing the liveness test, so GTM answers with all of them, or none.",
note: "Ashburn lost its upstream link for half an hour. The legacy app has one datacenter, so GTM had nowhere else to send it."},
{s: spellLegacyDC, key: "akamai/traffic/400019", sev: "critical", href: "/akamai",
title: "Akamai: 30% of CP code 400019's requests are failing with 5xx",
detail: "Edge errors for the legacy property: its origin is the Ashburn server GTM can't reach."},
{s: spellLegacyDC, key: "akamai/server/legacy.corp.example.akadns.net/198.51.100.50", sev: "warning", href: "/akamai",
title: "198.51.100.50 in Ashburn is failing GTM's liveness test for legacy.corp.example.akadns.net"},
{s: spellReplicaBehind, key: "cloudflare/errors/shop.example", sev: "warning", href: "/cloudflare",
title: "shop.example: 7% of requests failing",
detail: "Most on checkout.shop.example: 503s while HAProxy had one PostgreSQL replica for the shop's reads.",
note: "fleet-0032 fell 40 MB behind during the nightly vacuum and HAProxy took it out; the one replica left couldn't carry checkout's reads. Moving the vacuum out of business hours."},
{s: spellAppExits, key: "container/fleet-0007/app-04", sev: "critical", host: "fleet-0007", href: "/containers",
title: "app-04 on fleet-0007: not running"},
{s: spellDRPaths, key: "cluster/fleet-0040:fa-dr-01/connection/fa-prod-01", sev: "critical", host: "fleet-0040:fa-dr-01", href: "/clusters",
title: "Replication to fa-prod-01 in fa-dr-01 (fleet-0040): connected, but its paths aren't redundant"},
{s: spellLBFlaps, key: "cloudflare/origin/site-1-web/fleet-0052", sev: "warning", host: "fleet-0052", href: "/cloudflare",
title: "fleet-0052 is failing site-1-web's health check"},
{s: spellLBFlaps, key: "akamai/server/api.shop.example.akadns.net/203.0.2.11", sev: "warning", host: "fleet-0101", href: "/akamai",
title: "203.0.2.11 in Singapore is failing GTM's liveness test for api.shop.example.akadns.net"},
{s: spellVaultEdgeDown, key: "node/obs-beacond:vault-edge/vault-edge-e3", sev: "critical", host: "obs-beacond:vault-edge",
href: "/workloads?source=obs-beacond%3Avault-edge&q=vault-edge-e3", title: "Server vault-edge-e3 in vault-edge (obs-beacond) is down"},
{s: spellCrashloop, key: "workload/cluster-00/ns-07/deployment/app-097", sev: "warning", host: "cluster-00",
href: "/workloads?source=cluster-00&ns=ns-07&q=app-097", title: "deployment/app-097 in ns-07 (cluster-00): 1 of 10 pods: 1 restarting (CrashLoopBackOff)"},
{s: spellLegacy502, key: "check/web-legacy", sev: "warning", href: "/checks", title: "legacy.example.com is down"},
{s: spellBlocklisted, key: "check/rbl-mx2", sev: "warning", href: "/checks", title: "mx2.example.com on block lists is down"},
{s: spellLateBackup, key: "backup/fleet-0050/received/fleet-0080/pg-dump", sev: "serious", host: "fleet-0050", href: "/backups",
title: "fleet-0080's pg-dump backup on fleet-0050 is 26 hours old"},
}
+14 -3
View File
@@ -74,9 +74,9 @@ func serveVaults(ctx context.Context, dir string) {
now := time.Now()
vaults := []*fakeVault{
newFakeVault("vault-prod", "1.21.4+ent", true, "pki_int", now, 300*24*time.Hour, 400, 2000,
[]fakeRaft{{"v1", "leader", "alive"}, {"v2", "voter", "alive"}, {"v3", "voter", "alive"}}, 1, 18_000),
[]fakeRaft{{"v1", "leader", "alive"}, {"v2", "voter", "alive"}, {"v3", "voter", "alive"}}, 1, vaultLeaseBase["vault-prod"]),
newFakeVault("vault-edge", "2.4.1", false, "pki_edge", now, 12*24*time.Hour, 40, 0,
[]fakeRaft{{"e1", "leader", "alive"}, {"e2", "voter", "alive"}, {"e3", "voter", "alive"}}, 1, 900),
[]fakeRaft{{"e1", "leader", "alive"}, {"e2", "voter", "alive"}, {"e3", "voter", "alive"}}, 1, vaultLeaseBase["vault-edge"]),
}
var yaml strings.Builder
yaml.WriteString("vault:\n")
@@ -230,7 +230,7 @@ func (v *fakeVault) ServeHTTP(w http.ResponseWriter, r *http.Request) {
case p == "auth/token/renew-self":
out = map[string]any{"auth": map[string]any{"lease_duration": 24 * 3600}}
case p == "sys/metrics":
l := v.leases + int(float64(v.leases)/10*math.Sin(float64(now.Unix())/3600))
l := vaultLeases(v.leases, now)
fmt.Fprintf(w, "# TYPE vault_expire_num_leases gauge\nvault_expire_num_leases %d\nvault_expire_num_irrevocable_leases 0\n", l)
if v.enterprise {
fmt.Fprint(w, "vault_audit_log_request_failure 0\nvault_audit_log_response_failure 0\n")
@@ -268,3 +268,14 @@ func (v *fakeVault) ServeHTTP(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(out)
}
// vaultLeaseBase is each fake Vault's usual number of leases.
var vaultLeaseBase = map[string]int{"vault-prod": 18_000, "vault-edge": 900}
// vaultLeases is how many leases a Vault holds at t: more in the working
// day, when people and jobs log in, with a little noise.
func vaultLeases(base int, t time.Time) int {
h := float64(t.UTC().Hour()) + float64(t.UTC().Minute())/60
day := math.Sin(2 * math.Pi * (h - 14) / 24) // UTC 20:00 is early afternoon in Phoenix
return base + int(float64(base)*(0.12*day+0.02*jitter("leases", t.Truncate(time.Minute))))
}