Every demo problem started at the same minute of each cycle and lasted the same time, so Recently resolved read like a timetable, and 30 days of history held about 3,700 incidents: app-04's container exiting 19 times a day for a month (#57). Each occurrence's start now moves by up to an eighth of its period, its length is 70% to 130% of the plan's, and half the periods have none, four in five at night in Phoenix. All of it is seeded by the spell and the period, so the simulator's parts still agree and a restarted demo carries on where the clock is. The month of history now holds about 1,400 incidents with a working day's shape, and in the Phoenix day there's never an hour with nothing going on (tested over three days). Callers that stepped through a spell by its period or worked out its end from its length use the occurrences instead: last, between. Closes #57.
195 lines
10 KiB
Go
195 lines
10 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"hash/fnv"
|
|
"time"
|
|
)
|
|
|
|
// What goes wrong in the demo, and when (#35). A real fleet is mostly
|
|
// fine: a few things stay wrong until someone gets to them, and the rest
|
|
// come and go. So the demo has
|
|
//
|
|
// - three stories, each a cause with its effects:
|
|
// 1. fa-prod-01 lost a drive: the array is degraded and its writes are
|
|
// slow while the drive's data is rebuilt (all the time);
|
|
// 2. checkout errors: about every 2 hours phx1-db03, a PostgreSQL replica,
|
|
// falls behind, HAProxy takes it out, and the shop's checkout answers
|
|
// with errors until it catches up (20 minutes);
|
|
// 3. the legacy datacenter: about every 4 hours its server stops answering
|
|
// Akamai's liveness test, GTM has nowhere to send the legacy property,
|
|
// and its CP code fails (30 minutes);
|
|
// - a few lasting warnings anyone would have: a failover nobody moved
|
|
// back, a weak key, a VM snapshot left behind, a datastore filling, a
|
|
// zone on Flexible SSL, a renewal waiting on someone;
|
|
// - problems that come and go: a unit that fails and is restarted, a
|
|
// container that exits, a check that flaps, a host update that takes
|
|
// a while, a late backup, an IBM i job waiting on an answer.
|
|
//
|
|
// Spells count from the Unix epoch, not from when the simulator started,
|
|
// so every part of it agrees on what's happening now, a restarted demo
|
|
// picks up where the clock is, and the history the demo starts with can
|
|
// be worked out for any time in the past.
|
|
|
|
// spell is when one problem happens: about once every every, about at
|
|
// into each period, for about on. A zero every means all the time.
|
|
//
|
|
// Not like a clock (#57): each period's start moves by up to an eighth of
|
|
// the period, its length is 70% to 130% of on, and half the periods have
|
|
// none, four in five at night in Phoenix, where the fleet's main site is.
|
|
// That still starts something every 20 minutes or so, while 30 days of
|
|
// history hold about 1,400 incidents rather than 3,700. All of it is
|
|
// seeded by the spell and the period, so it's the same whenever it's
|
|
// worked out.
|
|
type spell struct {
|
|
every, at, on time.Duration
|
|
}
|
|
|
|
// always is a problem that's there all the time.
|
|
var always = spell{}
|
|
|
|
// occurrence is one spell, from start to end.
|
|
type occurrence struct{ start, end time.Time }
|
|
|
|
var phoenix = time.FixedZone("MST", -7*3600)
|
|
|
|
// nth is the spell in period n, counting from the Unix epoch; false when
|
|
// that period has none.
|
|
func (s spell) nth(n int64) (occurrence, bool) {
|
|
h := fnv.New64a()
|
|
fmt.Fprintf(h, "%d %d %d %d", s.every, s.at, s.on, n)
|
|
x := h.Sum64() // FNV leaves short, alike inputs' high bits alike: mix them
|
|
x ^= x >> 30
|
|
x *= 0xbf58476d1ce4e5b9
|
|
x ^= x >> 27
|
|
x *= 0x94d049bb133111eb
|
|
x ^= x >> 31
|
|
frac := func(shift uint) float64 { return float64((x>>shift)&0xffff) / 0x10000 }
|
|
on := time.Duration(float64(s.on) * (0.7 + 0.6*frac(0)))
|
|
at := s.at + time.Duration((2*frac(16)-1)*float64(s.every/8))
|
|
at = max(0, min(at, s.every-on)) // inside its own period
|
|
start := time.Unix(0, n*int64(s.every)).Add(at)
|
|
skip := 0.5
|
|
if hr := start.In(phoenix).Hour(); hr < 7 || hr >= 22 {
|
|
skip = 0.8
|
|
}
|
|
if frac(32) < skip {
|
|
return occurrence{}, false
|
|
}
|
|
return occurrence{start, start.Add(on)}, true
|
|
}
|
|
|
|
func (s spell) period(t time.Time) int64 { return t.UnixNano() / int64(s.every) }
|
|
|
|
func (s spell) active(t time.Time) bool {
|
|
if s.every == 0 {
|
|
return true
|
|
}
|
|
o, ok := s.nth(s.period(t))
|
|
return ok && !t.Before(o.start) && t.Before(o.end)
|
|
}
|
|
|
|
// last is the spell going on at t, or the last one before it; for a
|
|
// problem that's always there, one that began long ago and hasn't ended.
|
|
func (s spell) last(t time.Time) occurrence {
|
|
if s.every == 0 {
|
|
return occurrence{t.Add(-9 * 24 * time.Hour), t.Add(100 * 365 * 24 * time.Hour)}
|
|
}
|
|
for n := s.period(t); ; n-- {
|
|
if o, ok := s.nth(n); ok && !o.start.After(t) {
|
|
return o
|
|
}
|
|
}
|
|
}
|
|
|
|
// began is when the current spell started, or the last one if it's over.
|
|
func (s spell) began(t time.Time) time.Time { return s.last(t).start }
|
|
|
|
// between is every spell that starts from from to to, oldest first.
|
|
func (s spell) between(from, to time.Time) []occurrence {
|
|
if s.every == 0 {
|
|
return nil
|
|
}
|
|
var out []occurrence
|
|
for n := s.period(from); n <= s.period(to); n++ {
|
|
if o, ok := s.nth(n); ok && !o.start.Before(from) && !o.start.After(to) {
|
|
out = append(out, o)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// The spells, by story. Periods aren't multiples of each other, so the
|
|
// mix on screen changes through the day.
|
|
var (
|
|
// Story 2: checkout errors.
|
|
spellReplicaBehind = spell{every: 2 * time.Hour, at: 40 * time.Minute, on: 20 * time.Minute}
|
|
// Story 3: the legacy datacenter.
|
|
spellLegacyDC = spell{every: 4 * time.Hour, at: 70 * time.Minute, on: 30 * time.Minute}
|
|
|
|
// Coming and going.
|
|
spellRedisFails = spell{every: 110 * time.Minute, at: 20 * time.Minute, on: 12 * time.Minute}
|
|
spellAppExits = spell{every: 75 * time.Minute, at: 5 * time.Minute, on: 8 * time.Minute}
|
|
spellLegacy502 = spell{every: 140 * time.Minute, at: 30 * time.Minute, on: 15 * time.Minute}
|
|
spellLBFlaps = spell{every: 100 * time.Minute, at: 60 * time.Minute, on: 15 * time.Minute}
|
|
spellCrashloop = spell{every: 150 * time.Minute, at: 15 * time.Minute, on: 20 * time.Minute}
|
|
spellESXiUpdate = spell{every: 4 * time.Hour, at: 10 * time.Minute, on: 40 * time.Minute}
|
|
spellESXiMemory = spell{every: 130 * time.Minute, at: 50 * time.Minute, on: 15 * time.Minute}
|
|
spellAHVUpdate = spell{every: 270 * time.Minute, at: 100 * time.Minute, on: 40 * time.Minute}
|
|
spellPVEHA = spell{every: 150 * time.Minute, at: 90 * time.Minute, on: 20 * time.Minute}
|
|
spellDRPaths = spell{every: 3 * time.Hour, at: 130 * time.Minute, on: 30 * time.Minute}
|
|
spellBladeEvac = spell{every: 6 * time.Hour, at: 2 * time.Hour, on: 60 * time.Minute}
|
|
spellVDIUpdate = spell{every: 5 * time.Hour, at: 4 * time.Hour, on: 45 * time.Minute}
|
|
spellInquiry = spell{every: 3 * time.Hour, at: 110 * time.Minute, on: 30 * time.Minute}
|
|
spellOrdersDown = spell{every: 200 * time.Minute, at: 80 * time.Minute, on: 25 * time.Minute}
|
|
spellVaultEdgeDown = spell{every: 3 * time.Hour, at: 150 * time.Minute, on: 20 * time.Minute}
|
|
spellBlocklisted = spell{every: 5 * time.Hour, at: 3 * time.Hour, on: 40 * time.Minute}
|
|
spellLateBackup = spell{every: 6 * time.Hour, at: 5 * time.Hour, on: 50 * time.Minute}
|
|
spellBackupShrank = spell{every: 6 * time.Hour, at: 2 * time.Hour, on: 50 * time.Minute}
|
|
spellSQLAgent = spell{every: 4 * time.Hour, at: 3 * time.Hour, on: 45 * time.Minute}
|
|
spellFSCrash = spell{every: 8 * time.Hour, at: 7 * time.Hour, on: time.Hour}
|
|
spellActivation = spell{every: 3 * time.Hour, at: 20 * time.Minute, on: 80 * time.Minute}
|
|
)
|
|
|
|
// past is a problem as obs-beacond titles it, for the incidents the demo's
|
|
// history starts with (history.go): one for each time its spell came and
|
|
// went before the demo started. The keys and titles are obs-beacond's
|
|
// own, so the past and the present read alike.
|
|
type past struct {
|
|
s spell
|
|
key, sev, title, host string
|
|
href, detail, note string // note: what someone wrote afterwards, on the first of the stories' incidents in a day
|
|
}
|
|
|
|
var pastProblems = []past{
|
|
{s: spellLegacyDC, key: "akamai/property/legacy.corp.example.akadns.net", sev: "critical", href: "/akamai",
|
|
title: "Akamai GTM: legacy.corp.example.akadns.net has no live datacenter",
|
|
detail: "Every datacenter's server is failing the liveness test, so GTM answers with all of them, or none.",
|
|
note: "Ashburn lost its upstream link for half an hour. The legacy app has one datacenter, so GTM had nowhere else to send it."},
|
|
{s: spellLegacyDC, key: "akamai/traffic/400019", sev: "critical", href: "/akamai",
|
|
title: "Akamai: 30% of CP code 400019's requests are failing with 5xx",
|
|
detail: "Edge errors for the legacy property: its origin is the Ashburn server GTM can't reach."},
|
|
{s: spellLegacyDC, key: "akamai/server/legacy.corp.example.akadns.net/198.51.100.50", sev: "warning", href: "/akamai",
|
|
title: "198.51.100.50 in Ashburn is failing GTM's liveness test for legacy.corp.example.akadns.net"},
|
|
{s: spellReplicaBehind, key: "cloudflare/errors/shop.example", sev: "warning", href: "/cloudflare",
|
|
title: "shop.example: 7% of requests failing",
|
|
detail: "Most on checkout.shop.example: 503s while HAProxy had one PostgreSQL replica for the shop's reads.",
|
|
note: hostName(pgLagging) + " fell 40 MB behind during the nightly vacuum and HAProxy took it out; the one replica left couldn't carry checkout's reads. Moving the vacuum out of business hours."},
|
|
{s: spellAppExits, key: "container/" + hostName(demoExited) + "/app-04", sev: "critical", host: hostName(demoExited), href: "/containers",
|
|
title: "app-04 on " + hostName(demoExited) + ": not running"},
|
|
{s: spellDRPaths, key: "cluster/" + storageCollector.name + ":fa-dr-01/connection/fa-prod-01", sev: "critical", host: storageCollector.name + ":fa-dr-01", href: "/clusters",
|
|
title: "Replication to fa-prod-01 in fa-dr-01 (" + storageCollector.name + "): connected, but its paths aren't redundant"},
|
|
{s: spellLBFlaps, key: "cloudflare/origin/site-1-web/" + hostName(52), sev: "warning", host: hostName(52), href: "/cloudflare",
|
|
title: hostName(52) + " is failing site-1-web's health check"},
|
|
{s: spellLBFlaps, key: "akamai/server/api.shop.example.akadns.net/203.0.2.11", sev: "warning", host: hostName(101), href: "/akamai",
|
|
title: "203.0.2.11 in Singapore is failing GTM's liveness test for api.shop.example.akadns.net"},
|
|
{s: spellVaultEdgeDown, key: "node/obs-beacond:vault-edge/vault-edge-e3", sev: "critical", host: "obs-beacond:vault-edge",
|
|
href: "/workloads?source=obs-beacond%3Avault-edge&q=vault-edge-e3", title: "Server vault-edge-e3 in vault-edge (obs-beacond) is down"},
|
|
{s: spellCrashloop, key: "workload/cluster-00/ns-07/deployment/app-097", sev: "warning", host: "cluster-00",
|
|
href: "/workloads?source=cluster-00&ns=ns-07&q=app-097", title: "deployment/app-097 in ns-07 (cluster-00): 1 of 10 pods: 1 restarting (CrashLoopBackOff)"},
|
|
{s: spellLegacy502, key: "check/web-legacy", sev: "warning", href: "/checks", title: "legacy.example.com is down"},
|
|
{s: spellBlocklisted, key: "check/rbl-mx2", sev: "warning", href: "/checks", title: "mx2.example.com on block lists is down"},
|
|
{s: spellLateBackup, key: "backup/" + hostName(50) + "/received/" + hostName(80) + "/pg-dump", sev: "serious", host: hostName(50), href: "/backups",
|
|
title: hostName(80) + "'s pg-dump backup on " + hostName(50) + " is 26 hours old"},
|
|
}
|