From cba12cb2f3ef81860e309d04c1427bd925b5aa54 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Sun, 15 Nov 2026 23:40:34 -0700 Subject: [PATCH] Demo: problems come round about on a schedule, not like a clock Every demo problem started at the same minute of each cycle and lasted the same time, so Recently resolved read like a timetable, and 30 days of history held about 3,700 incidents: app-04's container exiting 19 times a day for a month (#57). Each occurrence's start now moves by up to an eighth of its period, its length is 70% to 130% of the plan's, and half the periods have none, four in five at night in Phoenix. All of it is seeded by the spell and the period, so the simulator's parts still agree and a restarted demo carries on where the clock is. The month of history now holds about 1,400 incidents with a working day's shape, and in the Phoenix day there's never an hour with nothing going on (tested over three days). Callers that stepped through a spell by its period or worked out its end from its length use the occurrences instead: last, between. Closes #57. --- docs/demo.md | 10 ++-- scripts/fleet/cluster.go | 4 +- scripts/fleet/demo.go | 12 ++--- scripts/fleet/history.go | 27 ++++++---- scripts/fleet/scenario.go | 93 ++++++++++++++++++++++++++------ scripts/fleet/scenario_test.go | 96 +++++++++++++++++++++++++--------- scripts/fleet/windows.go | 17 +++--- 7 files changed, 189 insertions(+), 70 deletions(-) diff --git a/docs/demo.md b/docs/demo.md index bc59013..1d374ab 100644 --- a/docs/demo.md +++ b/docs/demo.md @@ -61,9 +61,13 @@ Most of the simulated fleet is healthy. What's wrong follows a plan (`scripts/fleet/scenario.go`, and "The demo fleet" in development.md): three stories with a cause and its effects, a few lasting warnings, and problems that come and go through the day, so incidents open, resolve and -show up under Recently resolved. Because the plan runs on the clock, two -visitors an hour apart see different problems, and a restarted demo shows -the same ones the old one would have. +show up under Recently resolved. Each comes round about on a schedule but +not like a clock: its start and length vary, and it skips about half its +turns, four in five at night in Phoenix, so the history has a working +day's shape, about 1,400 incidents a month rather than thousands, and in +the day there's never an hour with nothing going on. Because the plan +runs on the clock, two visitors an hour apart see different problems, and +a restarted demo shows the same ones the old one would have. The demo starts with a month behind it, worked out from the same plan and clock (`scripts/fleet/history.go`): every host's hourly CPU, memory and diff --git a/scripts/fleet/cluster.go b/scripts/fleet/cluster.go index 1eea7d6..c1479da 100644 --- a/scripts/fleet/cluster.go +++ b/scripts/fleet/cluster.go @@ -120,8 +120,8 @@ func (c *fakeCluster) play(now time.Time) { w := cloneWorkload(e.good) if on { e.bad(&w, e.s.began(now)) - } else if !e.good.Since.IsZero() && now.Sub(e.s.began(now)) < e.s.every { - w.Since = e.s.began(now).Add(e.s.on) // right again since the spell ended + } else if o := e.s.last(now); !e.good.Since.IsZero() && now.Sub(o.start) < e.s.every { + w.Since = o.end // right again since the spell ended } c.recs[e.id] = w c.changed[e.id] = true diff --git a/scripts/fleet/demo.go b/scripts/fleet/demo.go index 594408f..b9d2f79 100644 --- a/scripts/fleet/demo.go +++ b/scripts/fleet/demo.go @@ -361,11 +361,11 @@ func (d *demoHost) containers(r *report.Report, now time.Time) { if c.Name != "app-04" { continue } - if b := spellAppExits.began(now); spellAppExits.active(now) { - c.State, c.Status, c.Health = "exited", fmt.Sprintf("Exited (137) %s ago", fuzzy(now.Sub(b))), "" + if o := spellAppExits.last(now); spellAppExits.active(now) { + c.State, c.Status, c.Health = "exited", fmt.Sprintf("Exited (137) %s ago", fuzzy(now.Sub(o.start))), "" } else { c.State, c.Health = "running", "healthy" - c.Status = fmt.Sprintf("Up %s (healthy)", fuzzy(now.Sub(b.Add(spellAppExits.on)))) + c.Status = fmt.Sprintf("Up %s (healthy)", fuzzy(now.Sub(o.end))) } } r.Containers = &report.Containers{List: list} @@ -392,11 +392,11 @@ func (d *demoHost) stamp(r *report.Report, start, now time.Time) { for _, u := range demoUnits(d.n) { un := report.Unit{Name: u + ".service", Load: "loaded", Active: "active", Sub: "running", Since: start.UTC()} if d.n == demoFailedUnit && u == "redis-server" { - switch b := spellRedisFails.began(now); { + switch o := spellRedisFails.last(now); { case spellRedisFails.active(now): un.Active, un.Sub, un.Restarts, un.Since = "failed", "failed", 5, time.Time{} - case b.After(start): // restarted by hand when the spell ended - un.Since = b.Add(spellRedisFails.on).UTC() + case o.start.After(start): // restarted by hand when the spell ended + un.Since = o.end.UTC() } } r.Units = append(r.Units, un) diff --git a/scripts/fleet/history.go b/scripts/fleet/history.go index 5c7a731..248a756 100644 --- a/scripts/fleet/history.go +++ b/scripts/fleet/history.go @@ -89,9 +89,11 @@ func writeHistory(path, keyFile string, days int, f pastFleet, now time.Time) er state.Change{Entity: "svc/shop", At: from.Add(time.Second), Status: "warning"}) for _, fl := range pastFlaps { s := fl.s - for b := s.began(from).Add(s.every); b.Add(s.on).Before(now); b = b.Add(s.every) { - statuses = append(statuses, state.Change{Entity: fl.entity, At: b.Add(20 * time.Second), Status: fl.status}, - state.Change{Entity: fl.entity, At: b.Add(s.on + 20*time.Second), Status: fl.after}) + for _, o := range s.between(from, now) { + if o.end.Before(now) { + statuses = append(statuses, state.Change{Entity: fl.entity, At: o.start.Add(20 * time.Second), Status: fl.status}, + state.Change{Entity: fl.entity, At: o.end.Add(20 * time.Second), Status: fl.after}) + } } } slices.SortStableFunc(statuses, func(a, b state.Change) int { return a.At.Compare(b.At) }) @@ -245,9 +247,9 @@ func clusterHours(c *fakeCluster, from, few, to time.Time) []state.WorkloadHour // obs-beacond where the numbers carry on. func writePastIncidents(db *state.DB, from, now time.Time) (int, error) { type occ struct { - p past - began time.Time - first bool // the first of its kind on its day: it gets the note + p past + began, end time.Time + first bool // the first of its kind on its day: it gets the note } var all []occ for _, p := range pastProblems { @@ -255,9 +257,12 @@ func writePastIncidents(db *state.DB, from, now time.Time) (int, error) { continue } day := "" - for b := p.s.began(from).Add(p.s.every); b.Add(p.s.on).Before(now); b = b.Add(p.s.every) { - d := b.UTC().Format("2006-01-02") - all = append(all, occ{p, b, d != day}) + for _, o := range p.s.between(from, now) { + if !o.end.Before(now) { + continue // still going: the live demo opens it + } + d := o.start.UTC().Format("2006-01-02") + all = append(all, occ{p, o.start, o.end, d != day}) day = d } } @@ -268,10 +273,10 @@ func writePastIncidents(db *state.DB, from, now time.Time) (int, error) { // resolved within a minute of its end. opened := o.began.Add(time.Duration(10+seedInt(p.key, o.began, 40)) * time.Second) fired := opened.Add(2 * time.Minute) - resolved := o.began.Add(p.s.on + time.Duration(15+seedInt(p.key+" end", o.began, 45))*time.Second) + resolved := o.end.Add(time.Duration(15+seedInt(p.key+" end", o.began, 45)) * time.Second) inc := state.Incident{ID: fmt.Sprintf("B-%04d", k+1), Key: p.key, Severity: p.sev, Title: p.title, Detail: p.detail, Host: p.host, Href: p.href, Opened: opened, Fired: fired, Notified: fired, Resolved: resolved} - if p.sev == "critical" && p.s.on > 10*time.Minute { + if p.sev == "critical" && o.end.Sub(o.began) > 10*time.Minute { inc.AckBy, inc.AckAt = "On call", fired.Add(time.Duration(2+seedInt(p.key+" ack", o.began, 6))*time.Minute) } if err := db.PutIncident(inc); err != nil { diff --git a/scripts/fleet/scenario.go b/scripts/fleet/scenario.go index 2f2a823..eba7865 100644 --- a/scripts/fleet/scenario.go +++ b/scripts/fleet/scenario.go @@ -1,6 +1,10 @@ package main -import "time" +import ( + "fmt" + "hash/fnv" + "time" +) // What goes wrong in the demo, and when (#35). A real fleet is mostly // fine: a few things stay wrong until someone gets to them, and the rest @@ -9,10 +13,10 @@ import "time" // - three stories, each a cause with its effects: // 1. fa-prod-01 lost a drive: the array is degraded and its writes are // slow while the drive's data is rebuilt (all the time); -// 2. checkout errors: every 2 hours phx1-db03, a PostgreSQL replica, +// 2. checkout errors: about every 2 hours phx1-db03, a PostgreSQL replica, // falls behind, HAProxy takes it out, and the shop's checkout answers // with errors until it catches up (20 minutes); -// 3. the legacy datacenter: every 4 hours its server stops answering +// 3. the legacy datacenter: about every 4 hours its server stops answering // Akamai's liveness test, GTM has nowhere to send the legacy property, // and its CP code fails (30 minutes); // - a few lasting warnings anyone would have: a failover nobody moved @@ -27,8 +31,16 @@ import "time" // picks up where the clock is, and the history the demo starts with can // be worked out for any time in the past. -// spell is when one problem is happening: for on, once every every, at -// into each period. A zero every means all the time. +// spell is when one problem happens: about once every every, about at +// into each period, for about on. A zero every means all the time. +// +// Not like a clock (#57): each period's start moves by up to an eighth of +// the period, its length is 70% to 130% of on, and half the periods have +// none, four in five at night in Phoenix, where the fleet's main site is. +// That still starts something every 20 minutes or so, while 30 days of +// history hold about 1,400 incidents rather than 3,700. All of it is +// seeded by the spell and the period, so it's the same whenever it's +// worked out. type spell struct { every, at, on time.Duration } @@ -36,26 +48,75 @@ type spell struct { // always is a problem that's there all the time. var always = spell{} +// occurrence is one spell, from start to end. +type occurrence struct{ start, end time.Time } + +var phoenix = time.FixedZone("MST", -7*3600) + +// nth is the spell in period n, counting from the Unix epoch; false when +// that period has none. +func (s spell) nth(n int64) (occurrence, bool) { + h := fnv.New64a() + fmt.Fprintf(h, "%d %d %d %d", s.every, s.at, s.on, n) + x := h.Sum64() // FNV leaves short, alike inputs' high bits alike: mix them + x ^= x >> 30 + x *= 0xbf58476d1ce4e5b9 + x ^= x >> 27 + x *= 0x94d049bb133111eb + x ^= x >> 31 + frac := func(shift uint) float64 { return float64((x>>shift)&0xffff) / 0x10000 } + on := time.Duration(float64(s.on) * (0.7 + 0.6*frac(0))) + at := s.at + time.Duration((2*frac(16)-1)*float64(s.every/8)) + at = max(0, min(at, s.every-on)) // inside its own period + start := time.Unix(0, n*int64(s.every)).Add(at) + skip := 0.5 + if hr := start.In(phoenix).Hour(); hr < 7 || hr >= 22 { + skip = 0.8 + } + if frac(32) < skip { + return occurrence{}, false + } + return occurrence{start, start.Add(on)}, true +} + +func (s spell) period(t time.Time) int64 { return t.UnixNano() / int64(s.every) } + func (s spell) active(t time.Time) bool { if s.every == 0 { return true } - p := time.Duration(t.UnixNano()) % s.every - return p >= s.at && p < s.at+s.on + o, ok := s.nth(s.period(t)) + return ok && !t.Before(o.start) && t.Before(o.end) } -// began is when the current spell started, or the last one if it's over; -// a long time ago for a problem that's always there. -func (s spell) began(t time.Time) time.Time { +// last is the spell going on at t, or the last one before it; for a +// problem that's always there, one that began long ago and hasn't ended. +func (s spell) last(t time.Time) occurrence { if s.every == 0 { - return t.Add(-9 * 24 * time.Hour) + return occurrence{t.Add(-9 * 24 * time.Hour), t.Add(100 * 365 * 24 * time.Hour)} } - p := time.Duration(t.UnixNano()) % s.every - start := t.Add(-p + s.at) - if start.After(t) { - start = start.Add(-s.every) + for n := s.period(t); ; n-- { + if o, ok := s.nth(n); ok && !o.start.After(t) { + return o + } } - return start +} + +// began is when the current spell started, or the last one if it's over. +func (s spell) began(t time.Time) time.Time { return s.last(t).start } + +// between is every spell that starts from from to to, oldest first. +func (s spell) between(from, to time.Time) []occurrence { + if s.every == 0 { + return nil + } + var out []occurrence + for n := s.period(from); n <= s.period(to); n++ { + if o, ok := s.nth(n); ok && !o.start.Before(from) && !o.start.After(to) { + out = append(out, o) + } + } + return out } // The spells, by story. Periods aren't multiples of each other, so the diff --git a/scripts/fleet/scenario_test.go b/scripts/fleet/scenario_test.go index 6a1cb57..8047370 100644 --- a/scripts/fleet/scenario_test.go +++ b/scripts/fleet/scenario_test.go @@ -7,33 +7,67 @@ import ( "github.com/cairnobs/cairnobs/internal/report" ) +// A spell comes round about every period, never like a clock (#57), and +// active, last and between always agree. func TestSpell(t *testing.T) { s := spell{every: time.Hour, at: 20 * time.Minute, on: 10 * time.Minute} - day := time.Date(2026, 11, 16, 0, 0, 0, 0, time.UTC) - for _, c := range []struct { - at time.Duration - active bool - began time.Duration - }{ - {19 * time.Minute, false, -40 * time.Minute}, - {20 * time.Minute, true, 20 * time.Minute}, - {29 * time.Minute, true, 20 * time.Minute}, - {30 * time.Minute, false, 20 * time.Minute}, - {80 * time.Minute, true, 80 * time.Minute}, - } { - now := day.Add(c.at) - if s.active(now) != c.active || !s.began(now).Equal(day.Add(c.began)) { - t.Errorf("at %s: active %v began %s", c.at, s.active(now), s.began(now).Sub(day)) + day := time.Date(2026, 11, 16, 7, 0, 0, 0, time.UTC) // midnight in Phoenix + week := s.between(day, day.Add(7*24*time.Hour)) + starts := map[time.Duration]bool{} + night, nights, daytime, days := 0, 0, 0, 0 + for i, o := range week { + if l := o.end.Sub(o.start); l < 7*time.Minute || l > 13*time.Minute { + t.Errorf("%s lasts %s", o.start, l) + } + if s.period(o.start) != s.period(o.end.Add(-time.Nanosecond)) { + t.Errorf("%s runs into the next period", o.start) + } + if i > 0 && !week[i-1].end.Before(o.start) { + t.Errorf("%s overlaps the one before", o.start) + } + starts[time.Duration(o.start.UnixNano())%s.every/time.Minute] = true + mid := o.start.Add(o.end.Sub(o.start) / 2) + if !s.active(mid) || !s.last(mid).start.Equal(o.start) || !s.began(o.end).Equal(o.start) || s.active(o.end) { + t.Errorf("%s: active %v last %s", mid, s.active(mid), s.last(mid).start) + } + if hr := o.start.In(phoenix).Hour(); hr < 7 || hr >= 22 { + night++ + } else { + daytime++ } } - if !always.active(day) || !always.began(day).Before(day) { + for h := 0; h < 7*24; h++ { + if hr := day.Add(time.Duration(h) * time.Hour).In(phoenix).Hour(); hr < 7 || hr >= 22 { + nights++ + } else { + days++ + } + } + if n := len(week); n < 7*24/5 || n > 7*24*2/3 { + t.Errorf("%d of %d hours had it", n, 7*24) + } + if float64(night)/float64(nights) >= float64(daytime)/float64(days) { + t.Errorf("as often at night (%d/%d) as by day (%d/%d)", night, nights, daytime, days) + } + if len(starts) < 5 { + t.Errorf("starts at only %d minutes of the hour", len(starts)) + } + // Between spells, the last one is the one before. + for i := 1; i < len(week); i++ { + gap := week[i-1].end.Add(time.Second) + if gap.Before(week[i].start) && (s.active(gap) || !s.last(gap).start.Equal(week[i-1].start)) { + t.Errorf("%s: active %v last %s", gap, s.active(gap), s.last(gap).start) + } + } + if !always.active(day) || !always.began(day).Before(day) || always.between(day, day.Add(time.Hour)) != nil { t.Error("always") } } -// The demo stays calm: across a day, no more than a few of the problems -// that come and go are on at once, and every one of them is on some of -// the time and off the rest. +// The demo stays calm: across three days, no more than a few of the +// problems that come and go are on at once, every one of them is on some +// of the time and off the rest, and in the Phoenix day a visitor never +// goes an hour without one. func TestScenarioIsCalm(t *testing.T) { spells := map[string]spell{"replica": spellReplicaBehind, "legacy dc": spellLegacyDC, "redis": spellRedisFails, "app": spellAppExits, "502": spellLegacy502, "lb": spellLBFlaps, "crashloop": spellCrashloop, "esxi update": spellESXiUpdate, "esxi memory": spellESXiMemory, @@ -43,7 +77,9 @@ func TestScenarioIsCalm(t *testing.T) { on := map[string]int{} most, total, steps := 0, 0, 0 day := time.Date(2026, 11, 16, 0, 0, 0, 0, time.UTC) - for m := 0; m < 24*60; m++ { + const minutes = 3 * 24 * 60 + quiet, longest := 0, 0 + for m := 0; m < minutes; m++ { now := day.Add(time.Duration(m) * time.Minute) n := 0 for name, s := range spells { @@ -56,12 +92,22 @@ func TestScenarioIsCalm(t *testing.T) { } } most, total, steps = max(most, n), total+n, steps+1 + if hr := now.In(phoenix).Hour(); n == 0 && hr >= 7 && hr < 22 { + quiet++ + longest = max(longest, quiet) + } else { + quiet = 0 + } } for name := range spells { - if on[name] == 0 || on[name] == 24*60 { - t.Errorf("%s is on %d minutes of the day", name, on[name]) + if on[name] == 0 || on[name] == minutes { + t.Errorf("%s is on %d minutes of three days", name, on[name]) } } + t.Logf("longest quiet daytime stretch %d min; %.1f on at once on average, %d at most", longest, float64(total)/float64(steps), most) + if longest >= 60 { + t.Errorf("%d minutes in the day with nothing going on", longest) + } if avg := float64(total) / float64(steps); avg > 4 || most > 9 { t.Errorf("on at once: %.1f on average, %d at most", avg, most) } @@ -82,13 +128,13 @@ func TestFlapSet(t *testing.T) { if len(c.episodes) != 2 { t.Fatalf("%d episodes, want a and q", len(c.episodes)) } - hour := time.Date(2026, 11, 16, 3, 0, 0, 0, time.UTC) - c.play(hour.Add(5 * time.Minute)) + o := s.between(time.Date(2026, 11, 16, 3, 0, 0, 0, time.UTC), time.Date(2026, 11, 17, 3, 0, 0, 0, time.UTC))[0] + c.play(o.start.Add(time.Minute)) if c.recs["a"].State != report.StateDown || c.recs["q"].ID != "q" { t.Fatalf("in the spell: %+v %+v", c.recs["a"], c.recs["q"]) } c.removed = nil - c.play(hour.Add(40 * time.Minute)) + c.play(o.end.Add(time.Minute)) if _, there := c.recs["q"]; c.recs["a"].State != report.StateOK || there || len(c.removed) != 1 { t.Fatalf("after: %+v, q there %v, removed %v", c.recs["a"], there, c.removed) } diff --git a/scripts/fleet/windows.go b/scripts/fleet/windows.go index 838e5f2..8608226 100644 --- a/scripts/fleet/windows.go +++ b/scripts/fleet/windows.go @@ -74,13 +74,16 @@ func newWindowsHost(name string, interval time.Duration) *host { // them. func (wh *winHost) stamp(h *host, r *report.Report, now time.Time) { w := wh.w - crashes, lastCrash := 0, time.Time{} + crashes, lastCrash, firstCrash := 0, time.Time{}, time.Time{} if w.name == "win-fs01" { // It goes down without shutting down cleanly and is straight back: // up since the last time. lastCrash = spellFSCrash.began(now) - for t := lastCrash; now.Sub(t) < 24*time.Hour; t = t.Add(-spellFSCrash.every) { - crashes++ + day := spellFSCrash.between(now.Add(-24*time.Hour+time.Nanosecond), now) + crashes = max(1, len(day)) + firstCrash = lastCrash + if len(day) > 0 { + firstCrash = day[0].start } h.start = lastCrash r.Uptime = max(0, now.Sub(h.start).Seconds()) @@ -89,10 +92,10 @@ func (wh *winHost) stamp(h *host, r *report.Report, now time.Time) { for _, u := range w.units { un := report.Unit{Name: u + ".service", Load: "loaded", Active: "active", Sub: "running", Since: h.start.UTC().Add(time.Minute).Truncate(time.Second)} if w.name == "win-sql01" && u == "SQLSERVERAGENT" { - if b := spellSQLAgent.began(now); spellSQLAgent.active(now) { + if o := spellSQLAgent.last(now); spellSQLAgent.active(now) { un.Active, un.Sub, un.Since = "failed", "dead", time.Time{} - } else if b.After(h.start) { // someone started it again - un.Since = b.Add(spellSQLAgent.on).UTC().Truncate(time.Second) + } else if o.start.After(h.start) { // someone started it again + un.Since = o.end.UTC().Truncate(time.Second) } } r.Units = append(r.Units, un) @@ -104,7 +107,7 @@ func (wh *winHost) stamp(h *host, r *report.Report, now time.Time) { } switch w.name { case "win-fs01": - first := lastCrash.Add(-time.Duration(crashes-1) * spellFSCrash.every) + first := firstCrash group("System", "Microsoft-Windows-Kernel-Power", 41, "critical", crashes, now.Sub(first), now.Sub(h.start), "The system has rebooted without cleanly shutting down first. This error could be caused if the system stopped responding, crashed, or lost power unexpectedly.") group("System", "disk", 7, "error", 14, 19*time.Hour, 2*time.Hour, `The device, \Device\Harddisk1\DR1, has a bad block.`) -- 2.54.0