diff --git a/docs/demo.md b/docs/demo.md index bc59013..1d374ab 100644 --- a/docs/demo.md +++ b/docs/demo.md @@ -61,9 +61,13 @@ Most of the simulated fleet is healthy. What's wrong follows a plan (`scripts/fleet/scenario.go`, and "The demo fleet" in development.md): three stories with a cause and its effects, a few lasting warnings, and problems that come and go through the day, so incidents open, resolve and -show up under Recently resolved. Because the plan runs on the clock, two -visitors an hour apart see different problems, and a restarted demo shows -the same ones the old one would have. +show up under Recently resolved. Each comes round about on a schedule but +not like a clock: its start and length vary, and it skips about half its +turns, four in five at night in Phoenix, so the history has a working +day's shape, about 1,400 incidents a month rather than thousands, and in +the day there's never an hour with nothing going on. Because the plan +runs on the clock, two visitors an hour apart see different problems, and +a restarted demo shows the same ones the old one would have. The demo starts with a month behind it, worked out from the same plan and clock (`scripts/fleet/history.go`): every host's hourly CPU, memory and diff --git a/scripts/fleet/cluster.go b/scripts/fleet/cluster.go index 1eea7d6..c1479da 100644 --- a/scripts/fleet/cluster.go +++ b/scripts/fleet/cluster.go @@ -120,8 +120,8 @@ func (c *fakeCluster) play(now time.Time) { w := cloneWorkload(e.good) if on { e.bad(&w, e.s.began(now)) - } else if !e.good.Since.IsZero() && now.Sub(e.s.began(now)) < e.s.every { - w.Since = e.s.began(now).Add(e.s.on) // right again since the spell ended + } else if o := e.s.last(now); !e.good.Since.IsZero() && now.Sub(o.start) < e.s.every { + w.Since = o.end // right again since the spell ended } c.recs[e.id] = w c.changed[e.id] = true diff --git a/scripts/fleet/demo.go b/scripts/fleet/demo.go index 594408f..b9d2f79 100644 --- a/scripts/fleet/demo.go +++ b/scripts/fleet/demo.go @@ -361,11 +361,11 @@ func (d *demoHost) containers(r *report.Report, now time.Time) { if c.Name != "app-04" { continue } - if b := spellAppExits.began(now); spellAppExits.active(now) { - c.State, c.Status, c.Health = "exited", fmt.Sprintf("Exited (137) %s ago", fuzzy(now.Sub(b))), "" + if o := spellAppExits.last(now); spellAppExits.active(now) { + c.State, c.Status, c.Health = "exited", fmt.Sprintf("Exited (137) %s ago", fuzzy(now.Sub(o.start))), "" } else { c.State, c.Health = "running", "healthy" - c.Status = fmt.Sprintf("Up %s (healthy)", fuzzy(now.Sub(b.Add(spellAppExits.on)))) + c.Status = fmt.Sprintf("Up %s (healthy)", fuzzy(now.Sub(o.end))) } } r.Containers = &report.Containers{List: list} @@ -392,11 +392,11 @@ func (d *demoHost) stamp(r *report.Report, start, now time.Time) { for _, u := range demoUnits(d.n) { un := report.Unit{Name: u + ".service", Load: "loaded", Active: "active", Sub: "running", Since: start.UTC()} if d.n == demoFailedUnit && u == "redis-server" { - switch b := spellRedisFails.began(now); { + switch o := spellRedisFails.last(now); { case spellRedisFails.active(now): un.Active, un.Sub, un.Restarts, un.Since = "failed", "failed", 5, time.Time{} - case b.After(start): // restarted by hand when the spell ended - un.Since = b.Add(spellRedisFails.on).UTC() + case o.start.After(start): // restarted by hand when the spell ended + un.Since = o.end.UTC() } } r.Units = append(r.Units, un) diff --git a/scripts/fleet/history.go b/scripts/fleet/history.go index 5c7a731..248a756 100644 --- a/scripts/fleet/history.go +++ b/scripts/fleet/history.go @@ -89,9 +89,11 @@ func writeHistory(path, keyFile string, days int, f pastFleet, now time.Time) er state.Change{Entity: "svc/shop", At: from.Add(time.Second), Status: "warning"}) for _, fl := range pastFlaps { s := fl.s - for b := s.began(from).Add(s.every); b.Add(s.on).Before(now); b = b.Add(s.every) { - statuses = append(statuses, state.Change{Entity: fl.entity, At: b.Add(20 * time.Second), Status: fl.status}, - state.Change{Entity: fl.entity, At: b.Add(s.on + 20*time.Second), Status: fl.after}) + for _, o := range s.between(from, now) { + if o.end.Before(now) { + statuses = append(statuses, state.Change{Entity: fl.entity, At: o.start.Add(20 * time.Second), Status: fl.status}, + state.Change{Entity: fl.entity, At: o.end.Add(20 * time.Second), Status: fl.after}) + } } } slices.SortStableFunc(statuses, func(a, b state.Change) int { return a.At.Compare(b.At) }) @@ -245,9 +247,9 @@ func clusterHours(c *fakeCluster, from, few, to time.Time) []state.WorkloadHour // obs-beacond where the numbers carry on. func writePastIncidents(db *state.DB, from, now time.Time) (int, error) { type occ struct { - p past - began time.Time - first bool // the first of its kind on its day: it gets the note + p past + began, end time.Time + first bool // the first of its kind on its day: it gets the note } var all []occ for _, p := range pastProblems { @@ -255,9 +257,12 @@ func writePastIncidents(db *state.DB, from, now time.Time) (int, error) { continue } day := "" - for b := p.s.began(from).Add(p.s.every); b.Add(p.s.on).Before(now); b = b.Add(p.s.every) { - d := b.UTC().Format("2006-01-02") - all = append(all, occ{p, b, d != day}) + for _, o := range p.s.between(from, now) { + if !o.end.Before(now) { + continue // still going: the live demo opens it + } + d := o.start.UTC().Format("2006-01-02") + all = append(all, occ{p, o.start, o.end, d != day}) day = d } } @@ -268,10 +273,10 @@ func writePastIncidents(db *state.DB, from, now time.Time) (int, error) { // resolved within a minute of its end. opened := o.began.Add(time.Duration(10+seedInt(p.key, o.began, 40)) * time.Second) fired := opened.Add(2 * time.Minute) - resolved := o.began.Add(p.s.on + time.Duration(15+seedInt(p.key+" end", o.began, 45))*time.Second) + resolved := o.end.Add(time.Duration(15+seedInt(p.key+" end", o.began, 45)) * time.Second) inc := state.Incident{ID: fmt.Sprintf("B-%04d", k+1), Key: p.key, Severity: p.sev, Title: p.title, Detail: p.detail, Host: p.host, Href: p.href, Opened: opened, Fired: fired, Notified: fired, Resolved: resolved} - if p.sev == "critical" && p.s.on > 10*time.Minute { + if p.sev == "critical" && o.end.Sub(o.began) > 10*time.Minute { inc.AckBy, inc.AckAt = "On call", fired.Add(time.Duration(2+seedInt(p.key+" ack", o.began, 6))*time.Minute) } if err := db.PutIncident(inc); err != nil { diff --git a/scripts/fleet/scenario.go b/scripts/fleet/scenario.go index 2f2a823..eba7865 100644 --- a/scripts/fleet/scenario.go +++ b/scripts/fleet/scenario.go @@ -1,6 +1,10 @@ package main -import "time" +import ( + "fmt" + "hash/fnv" + "time" +) // What goes wrong in the demo, and when (#35). A real fleet is mostly // fine: a few things stay wrong until someone gets to them, and the rest @@ -9,10 +13,10 @@ import "time" // - three stories, each a cause with its effects: // 1. fa-prod-01 lost a drive: the array is degraded and its writes are // slow while the drive's data is rebuilt (all the time); -// 2. checkout errors: every 2 hours phx1-db03, a PostgreSQL replica, +// 2. checkout errors: about every 2 hours phx1-db03, a PostgreSQL replica, // falls behind, HAProxy takes it out, and the shop's checkout answers // with errors until it catches up (20 minutes); -// 3. the legacy datacenter: every 4 hours its server stops answering +// 3. the legacy datacenter: about every 4 hours its server stops answering // Akamai's liveness test, GTM has nowhere to send the legacy property, // and its CP code fails (30 minutes); // - a few lasting warnings anyone would have: a failover nobody moved @@ -27,8 +31,16 @@ import "time" // picks up where the clock is, and the history the demo starts with can // be worked out for any time in the past. -// spell is when one problem is happening: for on, once every every, at -// into each period. A zero every means all the time. +// spell is when one problem happens: about once every every, about at +// into each period, for about on. A zero every means all the time. +// +// Not like a clock (#57): each period's start moves by up to an eighth of +// the period, its length is 70% to 130% of on, and half the periods have +// none, four in five at night in Phoenix, where the fleet's main site is. +// That still starts something every 20 minutes or so, while 30 days of +// history hold about 1,400 incidents rather than 3,700. All of it is +// seeded by the spell and the period, so it's the same whenever it's +// worked out. type spell struct { every, at, on time.Duration } @@ -36,26 +48,75 @@ type spell struct { // always is a problem that's there all the time. var always = spell{} +// occurrence is one spell, from start to end. +type occurrence struct{ start, end time.Time } + +var phoenix = time.FixedZone("MST", -7*3600) + +// nth is the spell in period n, counting from the Unix epoch; false when +// that period has none. +func (s spell) nth(n int64) (occurrence, bool) { + h := fnv.New64a() + fmt.Fprintf(h, "%d %d %d %d", s.every, s.at, s.on, n) + x := h.Sum64() // FNV leaves short, alike inputs' high bits alike: mix them + x ^= x >> 30 + x *= 0xbf58476d1ce4e5b9 + x ^= x >> 27 + x *= 0x94d049bb133111eb + x ^= x >> 31 + frac := func(shift uint) float64 { return float64((x>>shift)&0xffff) / 0x10000 } + on := time.Duration(float64(s.on) * (0.7 + 0.6*frac(0))) + at := s.at + time.Duration((2*frac(16)-1)*float64(s.every/8)) + at = max(0, min(at, s.every-on)) // inside its own period + start := time.Unix(0, n*int64(s.every)).Add(at) + skip := 0.5 + if hr := start.In(phoenix).Hour(); hr < 7 || hr >= 22 { + skip = 0.8 + } + if frac(32) < skip { + return occurrence{}, false + } + return occurrence{start, start.Add(on)}, true +} + +func (s spell) period(t time.Time) int64 { return t.UnixNano() / int64(s.every) } + func (s spell) active(t time.Time) bool { if s.every == 0 { return true } - p := time.Duration(t.UnixNano()) % s.every - return p >= s.at && p < s.at+s.on + o, ok := s.nth(s.period(t)) + return ok && !t.Before(o.start) && t.Before(o.end) } -// began is when the current spell started, or the last one if it's over; -// a long time ago for a problem that's always there. -func (s spell) began(t time.Time) time.Time { +// last is the spell going on at t, or the last one before it; for a +// problem that's always there, one that began long ago and hasn't ended. +func (s spell) last(t time.Time) occurrence { if s.every == 0 { - return t.Add(-9 * 24 * time.Hour) + return occurrence{t.Add(-9 * 24 * time.Hour), t.Add(100 * 365 * 24 * time.Hour)} } - p := time.Duration(t.UnixNano()) % s.every - start := t.Add(-p + s.at) - if start.After(t) { - start = start.Add(-s.every) + for n := s.period(t); ; n-- { + if o, ok := s.nth(n); ok && !o.start.After(t) { + return o + } } - return start +} + +// began is when the current spell started, or the last one if it's over. +func (s spell) began(t time.Time) time.Time { return s.last(t).start } + +// between is every spell that starts from from to to, oldest first. +func (s spell) between(from, to time.Time) []occurrence { + if s.every == 0 { + return nil + } + var out []occurrence + for n := s.period(from); n <= s.period(to); n++ { + if o, ok := s.nth(n); ok && !o.start.Before(from) && !o.start.After(to) { + out = append(out, o) + } + } + return out } // The spells, by story. Periods aren't multiples of each other, so the diff --git a/scripts/fleet/scenario_test.go b/scripts/fleet/scenario_test.go index 6a1cb57..8047370 100644 --- a/scripts/fleet/scenario_test.go +++ b/scripts/fleet/scenario_test.go @@ -7,33 +7,67 @@ import ( "github.com/cairnobs/cairnobs/internal/report" ) +// A spell comes round about every period, never like a clock (#57), and +// active, last and between always agree. func TestSpell(t *testing.T) { s := spell{every: time.Hour, at: 20 * time.Minute, on: 10 * time.Minute} - day := time.Date(2026, 11, 16, 0, 0, 0, 0, time.UTC) - for _, c := range []struct { - at time.Duration - active bool - began time.Duration - }{ - {19 * time.Minute, false, -40 * time.Minute}, - {20 * time.Minute, true, 20 * time.Minute}, - {29 * time.Minute, true, 20 * time.Minute}, - {30 * time.Minute, false, 20 * time.Minute}, - {80 * time.Minute, true, 80 * time.Minute}, - } { - now := day.Add(c.at) - if s.active(now) != c.active || !s.began(now).Equal(day.Add(c.began)) { - t.Errorf("at %s: active %v began %s", c.at, s.active(now), s.began(now).Sub(day)) + day := time.Date(2026, 11, 16, 7, 0, 0, 0, time.UTC) // midnight in Phoenix + week := s.between(day, day.Add(7*24*time.Hour)) + starts := map[time.Duration]bool{} + night, nights, daytime, days := 0, 0, 0, 0 + for i, o := range week { + if l := o.end.Sub(o.start); l < 7*time.Minute || l > 13*time.Minute { + t.Errorf("%s lasts %s", o.start, l) + } + if s.period(o.start) != s.period(o.end.Add(-time.Nanosecond)) { + t.Errorf("%s runs into the next period", o.start) + } + if i > 0 && !week[i-1].end.Before(o.start) { + t.Errorf("%s overlaps the one before", o.start) + } + starts[time.Duration(o.start.UnixNano())%s.every/time.Minute] = true + mid := o.start.Add(o.end.Sub(o.start) / 2) + if !s.active(mid) || !s.last(mid).start.Equal(o.start) || !s.began(o.end).Equal(o.start) || s.active(o.end) { + t.Errorf("%s: active %v last %s", mid, s.active(mid), s.last(mid).start) + } + if hr := o.start.In(phoenix).Hour(); hr < 7 || hr >= 22 { + night++ + } else { + daytime++ } } - if !always.active(day) || !always.began(day).Before(day) { + for h := 0; h < 7*24; h++ { + if hr := day.Add(time.Duration(h) * time.Hour).In(phoenix).Hour(); hr < 7 || hr >= 22 { + nights++ + } else { + days++ + } + } + if n := len(week); n < 7*24/5 || n > 7*24*2/3 { + t.Errorf("%d of %d hours had it", n, 7*24) + } + if float64(night)/float64(nights) >= float64(daytime)/float64(days) { + t.Errorf("as often at night (%d/%d) as by day (%d/%d)", night, nights, daytime, days) + } + if len(starts) < 5 { + t.Errorf("starts at only %d minutes of the hour", len(starts)) + } + // Between spells, the last one is the one before. + for i := 1; i < len(week); i++ { + gap := week[i-1].end.Add(time.Second) + if gap.Before(week[i].start) && (s.active(gap) || !s.last(gap).start.Equal(week[i-1].start)) { + t.Errorf("%s: active %v last %s", gap, s.active(gap), s.last(gap).start) + } + } + if !always.active(day) || !always.began(day).Before(day) || always.between(day, day.Add(time.Hour)) != nil { t.Error("always") } } -// The demo stays calm: across a day, no more than a few of the problems -// that come and go are on at once, and every one of them is on some of -// the time and off the rest. +// The demo stays calm: across three days, no more than a few of the +// problems that come and go are on at once, every one of them is on some +// of the time and off the rest, and in the Phoenix day a visitor never +// goes an hour without one. func TestScenarioIsCalm(t *testing.T) { spells := map[string]spell{"replica": spellReplicaBehind, "legacy dc": spellLegacyDC, "redis": spellRedisFails, "app": spellAppExits, "502": spellLegacy502, "lb": spellLBFlaps, "crashloop": spellCrashloop, "esxi update": spellESXiUpdate, "esxi memory": spellESXiMemory, @@ -43,7 +77,9 @@ func TestScenarioIsCalm(t *testing.T) { on := map[string]int{} most, total, steps := 0, 0, 0 day := time.Date(2026, 11, 16, 0, 0, 0, 0, time.UTC) - for m := 0; m < 24*60; m++ { + const minutes = 3 * 24 * 60 + quiet, longest := 0, 0 + for m := 0; m < minutes; m++ { now := day.Add(time.Duration(m) * time.Minute) n := 0 for name, s := range spells { @@ -56,12 +92,22 @@ func TestScenarioIsCalm(t *testing.T) { } } most, total, steps = max(most, n), total+n, steps+1 + if hr := now.In(phoenix).Hour(); n == 0 && hr >= 7 && hr < 22 { + quiet++ + longest = max(longest, quiet) + } else { + quiet = 0 + } } for name := range spells { - if on[name] == 0 || on[name] == 24*60 { - t.Errorf("%s is on %d minutes of the day", name, on[name]) + if on[name] == 0 || on[name] == minutes { + t.Errorf("%s is on %d minutes of three days", name, on[name]) } } + t.Logf("longest quiet daytime stretch %d min; %.1f on at once on average, %d at most", longest, float64(total)/float64(steps), most) + if longest >= 60 { + t.Errorf("%d minutes in the day with nothing going on", longest) + } if avg := float64(total) / float64(steps); avg > 4 || most > 9 { t.Errorf("on at once: %.1f on average, %d at most", avg, most) } @@ -82,13 +128,13 @@ func TestFlapSet(t *testing.T) { if len(c.episodes) != 2 { t.Fatalf("%d episodes, want a and q", len(c.episodes)) } - hour := time.Date(2026, 11, 16, 3, 0, 0, 0, time.UTC) - c.play(hour.Add(5 * time.Minute)) + o := s.between(time.Date(2026, 11, 16, 3, 0, 0, 0, time.UTC), time.Date(2026, 11, 17, 3, 0, 0, 0, time.UTC))[0] + c.play(o.start.Add(time.Minute)) if c.recs["a"].State != report.StateDown || c.recs["q"].ID != "q" { t.Fatalf("in the spell: %+v %+v", c.recs["a"], c.recs["q"]) } c.removed = nil - c.play(hour.Add(40 * time.Minute)) + c.play(o.end.Add(time.Minute)) if _, there := c.recs["q"]; c.recs["a"].State != report.StateOK || there || len(c.removed) != 1 { t.Fatalf("after: %+v, q there %v, removed %v", c.recs["a"], there, c.removed) } diff --git a/scripts/fleet/windows.go b/scripts/fleet/windows.go index 838e5f2..8608226 100644 --- a/scripts/fleet/windows.go +++ b/scripts/fleet/windows.go @@ -74,13 +74,16 @@ func newWindowsHost(name string, interval time.Duration) *host { // them. func (wh *winHost) stamp(h *host, r *report.Report, now time.Time) { w := wh.w - crashes, lastCrash := 0, time.Time{} + crashes, lastCrash, firstCrash := 0, time.Time{}, time.Time{} if w.name == "win-fs01" { // It goes down without shutting down cleanly and is straight back: // up since the last time. lastCrash = spellFSCrash.began(now) - for t := lastCrash; now.Sub(t) < 24*time.Hour; t = t.Add(-spellFSCrash.every) { - crashes++ + day := spellFSCrash.between(now.Add(-24*time.Hour+time.Nanosecond), now) + crashes = max(1, len(day)) + firstCrash = lastCrash + if len(day) > 0 { + firstCrash = day[0].start } h.start = lastCrash r.Uptime = max(0, now.Sub(h.start).Seconds()) @@ -89,10 +92,10 @@ func (wh *winHost) stamp(h *host, r *report.Report, now time.Time) { for _, u := range w.units { un := report.Unit{Name: u + ".service", Load: "loaded", Active: "active", Sub: "running", Since: h.start.UTC().Add(time.Minute).Truncate(time.Second)} if w.name == "win-sql01" && u == "SQLSERVERAGENT" { - if b := spellSQLAgent.began(now); spellSQLAgent.active(now) { + if o := spellSQLAgent.last(now); spellSQLAgent.active(now) { un.Active, un.Sub, un.Since = "failed", "dead", time.Time{} - } else if b.After(h.start) { // someone started it again - un.Since = b.Add(spellSQLAgent.on).UTC().Truncate(time.Second) + } else if o.start.After(h.start) { // someone started it again + un.Since = o.end.UTC().Truncate(time.Second) } } r.Units = append(r.Units, un) @@ -104,7 +107,7 @@ func (wh *winHost) stamp(h *host, r *report.Report, now time.Time) { } switch w.name { case "win-fs01": - first := lastCrash.Add(-time.Duration(crashes-1) * spellFSCrash.every) + first := firstCrash group("System", "Microsoft-Windows-Kernel-Power", 41, "critical", crashes, now.Sub(first), now.Sub(h.start), "The system has rebooted without cleanly shutting down first. This error could be caused if the system stopped responding, crashed, or lost power unexpectedly.") group("System", "disk", 7, "error", 14, 19*time.Hour, 2*time.Hour, `The device, \Device\Harddisk1\DR1, has a bad block.`)