Demo: problems come round about on a schedule, not like a clock #64

Merged
jcoffey-dev merged 1 commits from fix/demo-spells into main 2026-11-16 07:24:56 +00:00
7 changed files with 189 additions and 70 deletions

No files matched your search

+7 -3
View File
@@ -61,9 +61,13 @@ Most of the simulated fleet is healthy. What's wrong follows a plan
(`scripts/fleet/scenario.go`, and "The demo fleet" in development.md):
three stories with a cause and its effects, a few lasting warnings, and
problems that come and go through the day, so incidents open, resolve and
show up under Recently resolved. Because the plan runs on the clock, two
visitors an hour apart see different problems, and a restarted demo shows
the same ones the old one would have.
show up under Recently resolved. Each comes round about on a schedule but
not like a clock: its start and length vary, and it skips about half its
turns, four in five at night in Phoenix, so the history has a working
day's shape, about 1,400 incidents a month rather than thousands, and in
the day there's never an hour with nothing going on. Because the plan
runs on the clock, two visitors an hour apart see different problems, and
a restarted demo shows the same ones the old one would have.
The demo starts with a month behind it, worked out from the same plan and
clock (`scripts/fleet/history.go`): every host's hourly CPU, memory and
+2 -2
View File
@@ -120,8 +120,8 @@ func (c *fakeCluster) play(now time.Time) {
w := cloneWorkload(e.good)
if on {
e.bad(&w, e.s.began(now))
} else if !e.good.Since.IsZero() && now.Sub(e.s.began(now)) < e.s.every {
w.Since = e.s.began(now).Add(e.s.on) // right again since the spell ended
} else if o := e.s.last(now); !e.good.Since.IsZero() && now.Sub(o.start) < e.s.every {
w.Since = o.end // right again since the spell ended
}
c.recs[e.id] = w
c.changed[e.id] = true
+6 -6
View File
@@ -361,11 +361,11 @@ func (d *demoHost) containers(r *report.Report, now time.Time) {
if c.Name != "app-04" {
continue
}
if b := spellAppExits.began(now); spellAppExits.active(now) {
c.State, c.Status, c.Health = "exited", fmt.Sprintf("Exited (137) %s ago", fuzzy(now.Sub(b))), ""
if o := spellAppExits.last(now); spellAppExits.active(now) {
c.State, c.Status, c.Health = "exited", fmt.Sprintf("Exited (137) %s ago", fuzzy(now.Sub(o.start))), ""
} else {
c.State, c.Health = "running", "healthy"
c.Status = fmt.Sprintf("Up %s (healthy)", fuzzy(now.Sub(b.Add(spellAppExits.on))))
c.Status = fmt.Sprintf("Up %s (healthy)", fuzzy(now.Sub(o.end)))
}
}
r.Containers = &report.Containers{List: list}
@@ -392,11 +392,11 @@ func (d *demoHost) stamp(r *report.Report, start, now time.Time) {
for _, u := range demoUnits(d.n) {
un := report.Unit{Name: u + ".service", Load: "loaded", Active: "active", Sub: "running", Since: start.UTC()}
if d.n == demoFailedUnit && u == "redis-server" {
switch b := spellRedisFails.began(now); {
switch o := spellRedisFails.last(now); {
case spellRedisFails.active(now):
un.Active, un.Sub, un.Restarts, un.Since = "failed", "failed", 5, time.Time{}
case b.After(start): // restarted by hand when the spell ended
un.Since = b.Add(spellRedisFails.on).UTC()
case o.start.After(start): // restarted by hand when the spell ended
un.Since = o.end.UTC()
}
}
r.Units = append(r.Units, un)
+16 -11
View File
@@ -89,9 +89,11 @@ func writeHistory(path, keyFile string, days int, f pastFleet, now time.Time) er
state.Change{Entity: "svc/shop", At: from.Add(time.Second), Status: "warning"})
for _, fl := range pastFlaps {
s := fl.s
for b := s.began(from).Add(s.every); b.Add(s.on).Before(now); b = b.Add(s.every) {
statuses = append(statuses, state.Change{Entity: fl.entity, At: b.Add(20 * time.Second), Status: fl.status},
state.Change{Entity: fl.entity, At: b.Add(s.on + 20*time.Second), Status: fl.after})
for _, o := range s.between(from, now) {
if o.end.Before(now) {
statuses = append(statuses, state.Change{Entity: fl.entity, At: o.start.Add(20 * time.Second), Status: fl.status},
state.Change{Entity: fl.entity, At: o.end.Add(20 * time.Second), Status: fl.after})
}
}
}
slices.SortStableFunc(statuses, func(a, b state.Change) int { return a.At.Compare(b.At) })
@@ -245,9 +247,9 @@ func clusterHours(c *fakeCluster, from, few, to time.Time) []state.WorkloadHour
// obs-beacond where the numbers carry on.
func writePastIncidents(db *state.DB, from, now time.Time) (int, error) {
type occ struct {
p past
began time.Time
first bool // the first of its kind on its day: it gets the note
p past
began, end time.Time
first bool // the first of its kind on its day: it gets the note
}
var all []occ
for _, p := range pastProblems {
@@ -255,9 +257,12 @@ func writePastIncidents(db *state.DB, from, now time.Time) (int, error) {
continue
}
day := ""
for b := p.s.began(from).Add(p.s.every); b.Add(p.s.on).Before(now); b = b.Add(p.s.every) {
d := b.UTC().Format("2006-01-02")
all = append(all, occ{p, b, d != day})
for _, o := range p.s.between(from, now) {
if !o.end.Before(now) {
continue // still going: the live demo opens it
}
d := o.start.UTC().Format("2006-01-02")
all = append(all, occ{p, o.start, o.end, d != day})
day = d
}
}
@@ -268,10 +273,10 @@ func writePastIncidents(db *state.DB, from, now time.Time) (int, error) {
// resolved within a minute of its end.
opened := o.began.Add(time.Duration(10+seedInt(p.key, o.began, 40)) * time.Second)
fired := opened.Add(2 * time.Minute)
resolved := o.began.Add(p.s.on + time.Duration(15+seedInt(p.key+" end", o.began, 45))*time.Second)
resolved := o.end.Add(time.Duration(15+seedInt(p.key+" end", o.began, 45)) * time.Second)
inc := state.Incident{ID: fmt.Sprintf("B-%04d", k+1), Key: p.key, Severity: p.sev, Title: p.title, Detail: p.detail,
Host: p.host, Href: p.href, Opened: opened, Fired: fired, Notified: fired, Resolved: resolved}
if p.sev == "critical" && p.s.on > 10*time.Minute {
if p.sev == "critical" && o.end.Sub(o.began) > 10*time.Minute {
inc.AckBy, inc.AckAt = "On call", fired.Add(time.Duration(2+seedInt(p.key+" ack", o.began, 6))*time.Minute)
}
if err := db.PutIncident(inc); err != nil {
+77 -16
View File
@@ -1,6 +1,10 @@
package main
import "time"
import (
"fmt"
"hash/fnv"
"time"
)
// What goes wrong in the demo, and when (#35). A real fleet is mostly
// fine: a few things stay wrong until someone gets to them, and the rest
@@ -9,10 +13,10 @@ import "time"
// - three stories, each a cause with its effects:
// 1. fa-prod-01 lost a drive: the array is degraded and its writes are
// slow while the drive's data is rebuilt (all the time);
// 2. checkout errors: every 2 hours phx1-db03, a PostgreSQL replica,
// 2. checkout errors: about every 2 hours phx1-db03, a PostgreSQL replica,
// falls behind, HAProxy takes it out, and the shop's checkout answers
// with errors until it catches up (20 minutes);
// 3. the legacy datacenter: every 4 hours its server stops answering
// 3. the legacy datacenter: about every 4 hours its server stops answering
// Akamai's liveness test, GTM has nowhere to send the legacy property,
// and its CP code fails (30 minutes);
// - a few lasting warnings anyone would have: a failover nobody moved
@@ -27,8 +31,16 @@ import "time"
// picks up where the clock is, and the history the demo starts with can
// be worked out for any time in the past.
// spell is when one problem is happening: for on, once every every, at
// into each period. A zero every means all the time.
// spell is when one problem happens: about once every every, about at
// into each period, for about on. A zero every means all the time.
//
// Not like a clock (#57): each period's start moves by up to an eighth of
// the period, its length is 70% to 130% of on, and half the periods have
// none, four in five at night in Phoenix, where the fleet's main site is.
// That still starts something every 20 minutes or so, while 30 days of
// history hold about 1,400 incidents rather than 3,700. All of it is
// seeded by the spell and the period, so it's the same whenever it's
// worked out.
type spell struct {
every, at, on time.Duration
}
@@ -36,26 +48,75 @@ type spell struct {
// always is a problem that's there all the time.
var always = spell{}
// occurrence is one spell, from start to end.
type occurrence struct{ start, end time.Time }
var phoenix = time.FixedZone("MST", -7*3600)
// nth is the spell in period n, counting from the Unix epoch; false when
// that period has none.
func (s spell) nth(n int64) (occurrence, bool) {
h := fnv.New64a()
fmt.Fprintf(h, "%d %d %d %d", s.every, s.at, s.on, n)
x := h.Sum64() // FNV leaves short, alike inputs' high bits alike: mix them
x ^= x >> 30
x *= 0xbf58476d1ce4e5b9
x ^= x >> 27
x *= 0x94d049bb133111eb
x ^= x >> 31
frac := func(shift uint) float64 { return float64((x>>shift)&0xffff) / 0x10000 }
on := time.Duration(float64(s.on) * (0.7 + 0.6*frac(0)))
at := s.at + time.Duration((2*frac(16)-1)*float64(s.every/8))
at = max(0, min(at, s.every-on)) // inside its own period
start := time.Unix(0, n*int64(s.every)).Add(at)
skip := 0.5
if hr := start.In(phoenix).Hour(); hr < 7 || hr >= 22 {
skip = 0.8
}
if frac(32) < skip {
return occurrence{}, false
}
return occurrence{start, start.Add(on)}, true
}
func (s spell) period(t time.Time) int64 { return t.UnixNano() / int64(s.every) }
func (s spell) active(t time.Time) bool {
if s.every == 0 {
return true
}
p := time.Duration(t.UnixNano()) % s.every
return p >= s.at && p < s.at+s.on
o, ok := s.nth(s.period(t))
return ok && !t.Before(o.start) && t.Before(o.end)
}
// began is when the current spell started, or the last one if it's over;
// a long time ago for a problem that's always there.
func (s spell) began(t time.Time) time.Time {
// last is the spell going on at t, or the last one before it; for a
// problem that's always there, one that began long ago and hasn't ended.
func (s spell) last(t time.Time) occurrence {
if s.every == 0 {
return t.Add(-9 * 24 * time.Hour)
return occurrence{t.Add(-9 * 24 * time.Hour), t.Add(100 * 365 * 24 * time.Hour)}
}
p := time.Duration(t.UnixNano()) % s.every
start := t.Add(-p + s.at)
if start.After(t) {
start = start.Add(-s.every)
for n := s.period(t); ; n-- {
if o, ok := s.nth(n); ok && !o.start.After(t) {
return o
}
}
return start
}
// began is when the current spell started, or the last one if it's over.
func (s spell) began(t time.Time) time.Time { return s.last(t).start }
// between is every spell that starts from from to to, oldest first.
func (s spell) between(from, to time.Time) []occurrence {
if s.every == 0 {
return nil
}
var out []occurrence
for n := s.period(from); n <= s.period(to); n++ {
if o, ok := s.nth(n); ok && !o.start.Before(from) && !o.start.After(to) {
out = append(out, o)
}
}
return out
}
// The spells, by story. Periods aren't multiples of each other, so the
+71 -25
View File
@@ -7,33 +7,67 @@ import (
"github.com/cairnobs/cairnobs/internal/report"
)
// A spell comes round about every period, never like a clock (#57), and
// active, last and between always agree.
func TestSpell(t *testing.T) {
s := spell{every: time.Hour, at: 20 * time.Minute, on: 10 * time.Minute}
day := time.Date(2026, 11, 16, 0, 0, 0, 0, time.UTC)
for _, c := range []struct {
at time.Duration
active bool
began time.Duration
}{
{19 * time.Minute, false, -40 * time.Minute},
{20 * time.Minute, true, 20 * time.Minute},
{29 * time.Minute, true, 20 * time.Minute},
{30 * time.Minute, false, 20 * time.Minute},
{80 * time.Minute, true, 80 * time.Minute},
} {
now := day.Add(c.at)
if s.active(now) != c.active || !s.began(now).Equal(day.Add(c.began)) {
t.Errorf("at %s: active %v began %s", c.at, s.active(now), s.began(now).Sub(day))
day := time.Date(2026, 11, 16, 7, 0, 0, 0, time.UTC) // midnight in Phoenix
week := s.between(day, day.Add(7*24*time.Hour))
starts := map[time.Duration]bool{}
night, nights, daytime, days := 0, 0, 0, 0
for i, o := range week {
if l := o.end.Sub(o.start); l < 7*time.Minute || l > 13*time.Minute {
t.Errorf("%s lasts %s", o.start, l)
}
if s.period(o.start) != s.period(o.end.Add(-time.Nanosecond)) {
t.Errorf("%s runs into the next period", o.start)
}
if i > 0 && !week[i-1].end.Before(o.start) {
t.Errorf("%s overlaps the one before", o.start)
}
starts[time.Duration(o.start.UnixNano())%s.every/time.Minute] = true
mid := o.start.Add(o.end.Sub(o.start) / 2)
if !s.active(mid) || !s.last(mid).start.Equal(o.start) || !s.began(o.end).Equal(o.start) || s.active(o.end) {
t.Errorf("%s: active %v last %s", mid, s.active(mid), s.last(mid).start)
}
if hr := o.start.In(phoenix).Hour(); hr < 7 || hr >= 22 {
night++
} else {
daytime++
}
}
if !always.active(day) || !always.began(day).Before(day) {
for h := 0; h < 7*24; h++ {
if hr := day.Add(time.Duration(h) * time.Hour).In(phoenix).Hour(); hr < 7 || hr >= 22 {
nights++
} else {
days++
}
}
if n := len(week); n < 7*24/5 || n > 7*24*2/3 {
t.Errorf("%d of %d hours had it", n, 7*24)
}
if float64(night)/float64(nights) >= float64(daytime)/float64(days) {
t.Errorf("as often at night (%d/%d) as by day (%d/%d)", night, nights, daytime, days)
}
if len(starts) < 5 {
t.Errorf("starts at only %d minutes of the hour", len(starts))
}
// Between spells, the last one is the one before.
for i := 1; i < len(week); i++ {
gap := week[i-1].end.Add(time.Second)
if gap.Before(week[i].start) && (s.active(gap) || !s.last(gap).start.Equal(week[i-1].start)) {
t.Errorf("%s: active %v last %s", gap, s.active(gap), s.last(gap).start)
}
}
if !always.active(day) || !always.began(day).Before(day) || always.between(day, day.Add(time.Hour)) != nil {
t.Error("always")
}
}
// The demo stays calm: across a day, no more than a few of the problems
// that come and go are on at once, and every one of them is on some of
// the time and off the rest.
// The demo stays calm: across three days, no more than a few of the
// problems that come and go are on at once, every one of them is on some
// of the time and off the rest, and in the Phoenix day a visitor never
// goes an hour without one.
func TestScenarioIsCalm(t *testing.T) {
spells := map[string]spell{"replica": spellReplicaBehind, "legacy dc": spellLegacyDC, "redis": spellRedisFails, "app": spellAppExits,
"502": spellLegacy502, "lb": spellLBFlaps, "crashloop": spellCrashloop, "esxi update": spellESXiUpdate, "esxi memory": spellESXiMemory,
@@ -43,7 +77,9 @@ func TestScenarioIsCalm(t *testing.T) {
on := map[string]int{}
most, total, steps := 0, 0, 0
day := time.Date(2026, 11, 16, 0, 0, 0, 0, time.UTC)
for m := 0; m < 24*60; m++ {
const minutes = 3 * 24 * 60
quiet, longest := 0, 0
for m := 0; m < minutes; m++ {
now := day.Add(time.Duration(m) * time.Minute)
n := 0
for name, s := range spells {
@@ -56,12 +92,22 @@ func TestScenarioIsCalm(t *testing.T) {
}
}
most, total, steps = max(most, n), total+n, steps+1
if hr := now.In(phoenix).Hour(); n == 0 && hr >= 7 && hr < 22 {
quiet++
longest = max(longest, quiet)
} else {
quiet = 0
}
}
for name := range spells {
if on[name] == 0 || on[name] == 24*60 {
t.Errorf("%s is on %d minutes of the day", name, on[name])
if on[name] == 0 || on[name] == minutes {
t.Errorf("%s is on %d minutes of three days", name, on[name])
}
}
t.Logf("longest quiet daytime stretch %d min; %.1f on at once on average, %d at most", longest, float64(total)/float64(steps), most)
if longest >= 60 {
t.Errorf("%d minutes in the day with nothing going on", longest)
}
if avg := float64(total) / float64(steps); avg > 4 || most > 9 {
t.Errorf("on at once: %.1f on average, %d at most", avg, most)
}
@@ -82,13 +128,13 @@ func TestFlapSet(t *testing.T) {
if len(c.episodes) != 2 {
t.Fatalf("%d episodes, want a and q", len(c.episodes))
}
hour := time.Date(2026, 11, 16, 3, 0, 0, 0, time.UTC)
c.play(hour.Add(5 * time.Minute))
o := s.between(time.Date(2026, 11, 16, 3, 0, 0, 0, time.UTC), time.Date(2026, 11, 17, 3, 0, 0, 0, time.UTC))[0]
c.play(o.start.Add(time.Minute))
if c.recs["a"].State != report.StateDown || c.recs["q"].ID != "q" {
t.Fatalf("in the spell: %+v %+v", c.recs["a"], c.recs["q"])
}
c.removed = nil
c.play(hour.Add(40 * time.Minute))
c.play(o.end.Add(time.Minute))
if _, there := c.recs["q"]; c.recs["a"].State != report.StateOK || there || len(c.removed) != 1 {
t.Fatalf("after: %+v, q there %v, removed %v", c.recs["a"], there, c.removed)
}
+10 -7
View File
@@ -74,13 +74,16 @@ func newWindowsHost(name string, interval time.Duration) *host {
// them.
func (wh *winHost) stamp(h *host, r *report.Report, now time.Time) {
w := wh.w
crashes, lastCrash := 0, time.Time{}
crashes, lastCrash, firstCrash := 0, time.Time{}, time.Time{}
if w.name == "win-fs01" {
// It goes down without shutting down cleanly and is straight back:
// up since the last time.
lastCrash = spellFSCrash.began(now)
for t := lastCrash; now.Sub(t) < 24*time.Hour; t = t.Add(-spellFSCrash.every) {
crashes++
day := spellFSCrash.between(now.Add(-24*time.Hour+time.Nanosecond), now)
crashes = max(1, len(day))
firstCrash = lastCrash
if len(day) > 0 {
firstCrash = day[0].start
}
h.start = lastCrash
r.Uptime = max(0, now.Sub(h.start).Seconds())
@@ -89,10 +92,10 @@ func (wh *winHost) stamp(h *host, r *report.Report, now time.Time) {
for _, u := range w.units {
un := report.Unit{Name: u + ".service", Load: "loaded", Active: "active", Sub: "running", Since: h.start.UTC().Add(time.Minute).Truncate(time.Second)}
if w.name == "win-sql01" && u == "SQLSERVERAGENT" {
if b := spellSQLAgent.began(now); spellSQLAgent.active(now) {
if o := spellSQLAgent.last(now); spellSQLAgent.active(now) {
un.Active, un.Sub, un.Since = "failed", "dead", time.Time{}
} else if b.After(h.start) { // someone started it again
un.Since = b.Add(spellSQLAgent.on).UTC().Truncate(time.Second)
} else if o.start.After(h.start) { // someone started it again
un.Since = o.end.UTC().Truncate(time.Second)
}
}
r.Units = append(r.Units, un)
@@ -104,7 +107,7 @@ func (wh *winHost) stamp(h *host, r *report.Report, now time.Time) {
}
switch w.name {
case "win-fs01":
first := lastCrash.Add(-time.Duration(crashes-1) * spellFSCrash.every)
first := firstCrash
group("System", "Microsoft-Windows-Kernel-Power", 41, "critical", crashes, now.Sub(first), now.Sub(h.start),
"The system has rebooted without cleanly shutting down first. This error could be caused if the system stopped responding, crashed, or lost power unexpectedly.")
group("System", "disk", 7, "error", 14, 19*time.Hour, 2*time.Hour, `The device, \Device\Harddisk1\DR1, has a bad block.`)