From 3e5e2d59f9ab346500fa81ccfb0492e5eac28913 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Sun, 15 Nov 2026 22:34:01 -0700 Subject: [PATCH] Demo: a plan for what goes wrong, with problems that come and go The simulator made every problem it knew at once and kept it forever: 89 open alerts in the first minute, nothing ever resolved. Now scripts/fleet/scenario.go holds a plan: - three stories with a cause and its effects: fa-prod-01 lost a drive; a PostgreSQL replica falls behind, HAProxy takes it out and the shop's checkout fails; the legacy datacenter drops out of Akamai GTM and its CP code fails; - a few lasting warnings anyone would have; - problems that come and go (spells counted from the Unix epoch, so a restarted demo carries on where the clock is): a Redis unit, an exiting container, a flapping check and block listing, ESXi and AHV patching, a Proxmox HA error, Pure replication paths, a blade evacuating, an IBM i inquiry and an ended subsystem, vault-edge losing a server, a late or shrunken backup, SQL Server Agent stopping, win-fs01 crashing, an Akamai activation that drags. Cluster readers (Pure, vSphere, Nutanix, Proxmox, IBM i, the HMC, the Kubernetes clusters) flap records through fakeCluster.flap and flapSet; the HTTP stand-ins and host reports ask the spells directly. Gone: the fan at 0 RPM, the inaccessible VM, the crashed libvirt VM, the always- crashing pods, the ended and soon-ending Postgres certificates, a dozen expiring Vault and Akamai certificates, the stuck staging activation. A backup's folder is now named after the day the copy was made, so a late one no longer says it's from today (#43). Closes #35, closes #43. --- docs/demo.md | 10 +++ docs/development.md | 19 ++++-- scripts/fleet/akamai.go | 115 +++++++++++++++++++++----------- scripts/fleet/certs.go | 11 +--- scripts/fleet/cloudflare.go | 25 +++---- scripts/fleet/cluster.go | 117 +++++++++++++++++++++++++++++++-- scripts/fleet/datalayer.go | 11 +++- scripts/fleet/demo.go | 93 ++++++++++++++++++++------ scripts/fleet/hmc.go | 13 ++-- scripts/fleet/ibmi.go | 102 +++++++++++++++------------- scripts/fleet/main.go | 9 +-- scripts/fleet/nutanix.go | 12 +--- scripts/fleet/proxmox.go | 6 +- scripts/fleet/pure.go | 50 +++++++------- scripts/fleet/scenario.go | 91 +++++++++++++++++++++++++ scripts/fleet/scenario_test.go | 106 +++++++++++++++++++++++++++++ scripts/fleet/vault.go | 28 ++++---- scripts/fleet/vsphere.go | 17 +++-- scripts/fleet/windows.go | 35 ++++++---- 19 files changed, 640 insertions(+), 230 deletions(-) create mode 100644 scripts/fleet/scenario.go create mode 100644 scripts/fleet/scenario_test.go diff --git a/docs/demo.md b/docs/demo.md index 105da95..21a08ca 100644 --- a/docs/demo.md +++ b/docs/demo.md @@ -49,3 +49,13 @@ obs-beacond and the simulator, so nothing in it can reach anything else: - The root is read-only and the state is in tmpfs, so restarting the container is a fresh demo; restart it on a timer (weekly is plenty) so history doesn't pile up. + +## What a visitor sees + +Most of the simulated fleet is healthy. What's wrong follows a plan +(`scripts/fleet/scenario.go`, and "The demo fleet" in development.md): +three stories with a cause and its effects, a few lasting warnings, and +problems that come and go through the day, so incidents open, resolve and +show up under Recently resolved. Because the plan runs on the clock, two +visitors an hour apart see different problems, and a restarted demo shows +the same ones the old one would have. diff --git a/docs/development.md b/docs/development.md index 9d90d41..688ec24 100644 --- a/docs/development.md +++ b/docs/development.md @@ -67,11 +67,20 @@ send (ntfy, Discord, PagerDuty, a webhook and email) and lists it on http://127.0.0.1:58450/. `fleet run -h` lists the rest: clusters, VMs, Proxmox, vSphere, Nutanix and Pure arrays. -Some things are wrong on purpose, so every page has something to show: a -container exited, a unit failed, a stale backup, PostgreSQL's leader away -from home with a replica behind, an origin failing Cloudflare's health -check, a zone on Flexible SSL, and so on. The mail server is healthy, to -show what normal looks like. +Some things are wrong on purpose, so every page has something to show, +and most of the fleet is fine, as a real one is. `scripts/fleet/scenario.go` +has the plan: three stories with a cause and its effects (a FlashArray +lost a drive; a PostgreSQL replica falls behind and the shop's checkout +fails; the legacy datacenter drops out of Akamai GTM), a handful of +lasting warnings (a failover nobody moved back, a weak key, an old VM +snapshot, a zone on Flexible SSL, and so on), and problems that come and +go: a unit that fails and is restarted, a container that exits, a check +that flaps, a late backup, a host being patched. Each of those is a +*spell*: on for a while once every so often, counted from the Unix epoch, +so every part of the simulator agrees on what's happening now and a +restarted demo carries on where the clock is. Incidents open and resolve +through the day, and a dozen or two are open at any moment. The mail +server is healthy, to show what normal looks like. ## Dependencies and notices diff --git a/scripts/fleet/akamai.go b/scripts/fleet/akamai.go index aeb348e..8bd3a99 100644 --- a/scripts/fleet/akamai.go +++ b/scripts/fleet/akamai.go @@ -17,18 +17,19 @@ import ( // account on loopback, every signature checked. GTM's datacenters are // the first sites and its servers their load balancers' public // addresses (netmap.go, positions 1 and 2), so a failing one is a fleet -// host. Problems on purpose: +// host. Problems on purpose, the ones marked * coming and going +// (scenario.go): // -// GTM site 2's first load balancer failing for api.shop; the -// legacy property with no live datacenter; corp's change -// propagating for 45 minutes -// CPS 180 enrollments: four ending within 21 days (one within -// 7), two third-party renewals waiting on a CSR -// Property 320 properties: a failed production activation, one -// Manager stuck on staging, two rolling out +// GTM * site 2's first load balancer failing for api.shop; +// * the legacy property with no live datacenter (story 3) +// CPS 180 enrollments: a third-party renewal waiting on a CSR, +// ending in 15 days +// Property 320 properties: a failed production activation; +// Manager * one rolling out for longer than it should // Edge DNS 40 zones (one of 5,000 records); one changes an MX 20 // minutes after the start -// Traffic 320 CP codes; one at 8% 5xx, one at 30% +// Traffic 320 CP codes; * the legacy site's at 30% 5xx while its +// datacenter is down (story 3) const akamaiPort = 58410 func serveAkamai(ctx context.Context, dir string) { @@ -48,6 +49,18 @@ func serveAkamai(ctx context.Context, dir string) { f := &akamai.Fake{Creds: c} start := time.Now() n := fleetAkamai(f) + go func() { + t := time.NewTicker(30 * time.Second) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case now := <-t.C: + akamaiStory(f, now) + } + } + }() srv := &http.Server{Handler: f, ReadHeaderTimeout: 5 * time.Second} go func() { _ = srv.Serve(ln) }() go func() { <-ctx.Done(); _ = srv.Close() }() @@ -90,35 +103,25 @@ func fleetAkamai(f *akamai.Fake) string { switch name { case "shop": ps[0].Name = "api" - if t := ps[0].Targets[3100+2%sites]; t != nil { - t[lb(2%sites, 1)] = false - } case "corp": - ps = append(ps, akamai.FakeProperty{Name: "legacy", Type: "failover", Targets: map[int]map[string]bool{3100: {"198.51.100.50": false}}}) + ps = append(ps, akamai.FakeProperty{Name: "legacy", Type: "failover", Targets: map[int]map[string]bool{3100: {legacyServer: true}}}) } props += len(ps) - fd := akamai.FakeDomain{Type: "full", Datacenters: dcs, Properties: ps} - if name == "corp" { - fd.Propagation, fd.Since = "PENDING", now.Add(-45*time.Minute) - } - f.SetDomain(name+".example.akadns.net", fd) + f.SetDomain(name+".example.akadns.net", akamai.FakeDomain{Type: "full", Datacenters: dcs, Properties: ps}) } // CPS: ECDSA keys (fast to make); every tenth has an RSA one too. for i := range 180 { cn := fmt.Sprintf("app%03d.example.com", i) left := time.Duration(25+(i*37)%300) * day - switch i { - case 3: - left = 5 * day - case 11, 47, 101: - left = time.Duration(10+i%9) * day + if i == 47 { + left = 15 * day } e := akamai.FakeEnrollment{CN: cn, SANs: []string{cn, "www." + cn}, Validation: "dv", Contract: "1-FLEET", Waiting: "-"} issuer := "Let's Encrypt E7" if i%6 == 0 { e.Validation, issuer = "ov", "DigiCert Global G2 TLS RSA SHA256 2020 CA1" } - if i == 47 || i == 120 { + if i == 47 { e.Validation, e.Waiting, issuer = "third-party", "third-party-csr", "Sectigo RSA Domain Validation Secure Server CA" } e.Deployed = []string{akamai.FakePEM(issuer, e.SANs, true, now.Add(left))} @@ -142,15 +145,11 @@ func fleetAkamai(f *akamai.Fake) string { p.Latest, p.Staging = v+1, v+1 p.Activations = append(p.Activations, staged, akamai.FakeActivation{Version: v + 1, Network: "PRODUCTION", Status: "FAILED", Submitted: now.Add(-3 * time.Hour), Note: "Origin moved to the new load balancers"}) - case 9: - p.Latest = v + 1 - p.Activations = append(p.Activations, akamai.FakeActivation{Version: v + 1, Network: "STAGING", Status: "ZONE_1", Submitted: now.Add(-100 * time.Minute)}) - case 15, 16: - p.Latest, p.Staging = v+1, v+1 - p.Activations = append(p.Activations, staged, - akamai.FakeActivation{Version: v + 1, Network: "PRODUCTION", Status: "PENDING", Submitted: now.Add(-time.Duration(i-10) * time.Minute)}) } f.SetPMProperty(fmt.Sprintf("prp_9%05d", i), p) + if i == slowActivation { + slowBase = p + } } // Edge DNS. records := 0 @@ -164,21 +163,57 @@ func fleetAkamai(f *akamai.Fake) string { records += len(recs) f.SetZone(fmt.Sprintf("zone%02d.example", z), akamai.FakeZone{Records: recs}) } - // Traffic: a long tail; CP code 400007 at 8% 5xx, 400019 at 30%. + // Traffic: a long tail, about one request in 2,000 failing. for i := range 320 { hits := int64(2_000_000 / (i + 1)) - errs := hits / 2000 - switch i { - case 7: - errs = hits * 8 / 100 - case 19: - errs = hits * 30 / 100 - } - f.SetTraffic(int64(400000+i), hits, errs) + f.SetTraffic(int64(400000+i), hits, hits/2000) } + akamaiStory(f, now) return fmt.Sprintf("%d GTM properties over %d datacenters, 180 CPS enrollments, 320 properties, 40 DNS zones with %d record sets, 320 CP codes", props, sites, records) } +const ( + legacyServer = "198.51.100.50" // the legacy property's one datacenter's server + legacyCPCode = 400019 // the legacy site's traffic + slowActivation = 15 // the property whose production activation drags +) + +// slowBase is the slow property as fleetAkamai made it, before its new +// version. +var slowBase akamai.FakePMProperty + +// akamaiStory puts the problems that come and go in their state now +// (scenario.go): story 3, the legacy datacenter; a load balancer that +// fails GTM's liveness test now and then; an activation that drags. +func akamaiStory(f *akamai.Fake, now time.Time) { + legacyDown := spellLegacyDC.active(now) + f.SetAlive("corp.example.akadns.net", "legacy", 3100, legacyServer, !legacyDown) + hits := int64(2_000_000 / (legacyCPCode - 400000 + 1)) + errs := hits / 2000 + if legacyDown { + errs = hits * 30 / 100 + } + f.SetTraffic(legacyCPCode, hits, errs) + if sites := min(fleetSites, 6); sites > 2 { + f.SetAlive("shop.example.akadns.net", "api", 3100+2, fmt.Sprintf("203.0.%d.%d", 2, 11), !spellLBFlaps.active(now)) + } + // Its new version goes to production at the start of each spell and + // is active an hour and twenty minutes later. + p := slowBase + p.Activations = append([]akamai.FakeActivation{}, p.Activations...) + v := p.Latest + 1 + b := spellActivation.began(now) + p.Latest, p.Staging = v, v + p.Activations = append(p.Activations, akamai.FakeActivation{Version: v, Network: "STAGING", Status: "ACTIVE", Submitted: b.Add(-2 * time.Hour)}) + if spellActivation.active(now) { + p.Activations = append(p.Activations, akamai.FakeActivation{Version: v, Network: "PRODUCTION", Status: "PENDING", Submitted: b}) + } else { + p.Production = v + p.Activations = append(p.Activations, akamai.FakeActivation{Version: v, Network: "PRODUCTION", Status: "ACTIVE", Submitted: b}) + } + f.SetPMProperty(fmt.Sprintf("prp_9%05d", slowActivation), p) +} + // fleetZone is zone z's ordinary records, with the given MX. func fleetZone(z int, mx string) []akamai.RecordSet { name := fmt.Sprintf("zone%02d.example", z) diff --git a/scripts/fleet/certs.go b/scripts/fleet/certs.go index baec874..7ab8e93 100644 --- a/scripts/fleet/certs.go +++ b/scripts/fleet/certs.go @@ -19,8 +19,7 @@ import ( // last one though its file was renewed, and site 5's sends it // without its intermediate // 30-39 PostgreSQL: each its own certificate from the internal CA -// (STARTTLS), and in /etc/pki/tls/certs; a few end soon, one -// has a 1024-bit key +// (STARTTLS), and in /etc/pki/tls/certs; one has a 1024-bit key // // Clusters have the router's (or ingress controller's) wildcard on every // Route or Ingress, and cert-manager Certificates for ten of their apps. @@ -76,12 +75,6 @@ func hostCerts(i int, name string) (*report.CertScan, *report.CertFiles) { file("/etc/haproxy/certs/example.com.pem", c, wildcard.NotBefore.Add(time.Hour)) case pos >= 30 && pos < 40: begun := time.Duration(30+(n*37)%300) * 24 * time.Hour - switch { - case n%97 == 30: // ends in a few days - begun = 360 * 24 * time.Hour - case n%131 == 31: // ended - begun = 370 * 24 * time.Hour - } key := "ECDSA P-256" if n%211 == 32 { key = "RSA 1024" @@ -158,7 +151,7 @@ func clusterCerts(name string, ocp bool, routes int, put func(report.Workload)) for n := range min(10, routes) { app := appNames[n] host := app + ".example.com" - begun := time.Duration(10+(n*7+k*3)%60) * 24 * time.Hour + begun := time.Duration(10+(n*7+k*3)%55) * 24 * time.Hour // all a month or more from their end, renewing at 30 days if n == 4 && k == 0 { begun = 86 * 24 * time.Hour // ends in four days: the renewal is failing } diff --git a/scripts/fleet/cloudflare.go b/scripts/fleet/cloudflare.go index 03b1bf5..948f1df 100644 --- a/scripts/fleet/cloudflare.go +++ b/scripts/fleet/cloudflare.go @@ -17,11 +17,10 @@ import ( // dozen zones, a load-balanced pool per site whose origins are the sites' // load balancers, and the last 15 minutes of traffic. obs-beacond reads it // through cloudflare's stand_in (http on loopback only). Problems on -// purpose: legacy.example still uses Flexible SSL, partners.example's edge -// certificate ends in 9 days, media.example's advanced certificate is -// stuck validating, fleet-0052 fails its pool's health check from eastern -// North America, and 7% of shop.example's requests fail, most on -// checkout.shop.example. +// purpose: legacy.example still uses Flexible SSL; now and then fleet-0052 +// fails its pool's health check from eastern North America; and while +// the shop's database replica is behind (scenario.go, story 2) 7% of +// shop.example's requests fail, most on checkout.shop.example. const cloudflarePort = 58430 // cfToken is the fake's API token; not a secret, as the fake is loopback only. @@ -109,16 +108,8 @@ func cfHandle(w http.ResponseWriter, r *http.Request) { return } ends := now.Add(time.Duration(40+k*3) * 24 * time.Hour) - if z == "partners.example" { - ends = now.Add(9*24*time.Hour + 5*time.Hour) - } - packs := []map[string]any{{"type": "universal", "status": "active", "hosts": []string{z, "*." + z}, - "certificates": []map[string]string{{"expires_on": ends.Format(time.RFC3339)}}}} - if z == "media.example" { - packs = append(packs, map[string]any{"type": "advanced", "status": "pending_validation", "hosts": []string{"cdn." + z}, - "certificates": []map[string]string{}}) - } - ok(packs) + ok([]map[string]any{{"type": "universal", "status": "active", "hosts": []string{z, "*." + z}, + "certificates": []map[string]string{{"expires_on": ends.Format(time.RFC3339)}}}}) case len(parts) == 3 && parts[0] == "zones" && parts[2] == "load_balancers": _, z, found := zone(parts[1]) if !found { @@ -169,7 +160,7 @@ func cfHandle(w http.ResponseWriter, r *http.Request) { for pos := 1; pos <= 2; pos++ { _, a := cfOrigin(s, pos) h := map[string]any{"healthy": true, "rtt": fmt.Sprintf("%dms", 20+ri*30+pos*3), "failure_reason": "No failures", "response_code": 200} - if s == 1 && pos == 2 && region == "ENAM" { + if s == 1 && pos == 2 && region == "ENAM" && spellLBFlaps.active(now) { h = map[string]any{"healthy": false, "rtt": "0ms", "failure_reason": "Response code mismatch error", "response_code": 503} } origins = append(origins, map[string]any{a: h}) @@ -197,7 +188,7 @@ func cfHandle(w http.ResponseWriter, r *http.Request) { return map[string]any{"count": n, "dimensions": map[string]any{"edgeResponseStatus": status, "securityAction": action, "clientRequestHTTPHost": host}} } win := []map[string]any{g(base, 200, "unknown", z), g(base/20, 304, "unknown", "www."+z), g(3, 502, "unknown", z), g(base/400, 403, "block", z)} - if z == "shop.example" { + if z == "shop.example" && spellReplicaBehind.active(now) { win = append(win, g(base*6/100, 503, "unknown", "checkout.shop.example"), g(base/100, 502, "unknown", z)) } zs = append(zs, map[string]any{"zoneTag": id, "win": win, "day": []map[string]int{{"count": base / 400 * 96}}}) diff --git a/scripts/fleet/cluster.go b/scripts/fleet/cluster.go index 550e633..1eea7d6 100644 --- a/scripts/fleet/cluster.go +++ b/scripts/fleet/cluster.go @@ -16,6 +16,7 @@ import ( "math" mrand "math/rand/v2" "net/http" + "reflect" "slices" "strconv" "strings" @@ -47,6 +48,105 @@ type fakeCluster struct { noUsage bool // its reader sends none (Nutanix: usage is a separate API, not read yet) source string // what a host agent calls it (a vCenter's host name); "" for a cluster agent usageFn func(now time.Time) []report.Usage // its own readings instead of CPU and memory (an array's performance) + episodes []episode // records that go wrong and come right (scenario.go) +} + +// episode is a record that takes its bad form while its spell is on. +type episode struct { + id string + s spell + good report.Workload + bad func(w *report.Workload, since time.Time) + on, in bool // on: in its bad form; in: played at least once +} + +// flap makes record id go wrong during s and come right after +// (scenario.go): bad changes a copy of the record as it is now, given +// when the spell began. +func (c *fakeCluster) flap(id string, s spell, bad func(w *report.Workload, since time.Time)) { + if _, ok := c.recs[id]; !ok { + panic("flap: no record " + id) + } + c.episodes = append(c.episodes, episode{id: id, s: s, good: cloneWorkload(c.recs[id]), bad: bad}) + c.play(time.Now()) +} + +// flapSet flaps every record that bad has and good hasn't, or has +// otherwise: a reader's records made from a good and a bad snapshot. A +// record only bad has is there during the spell and gone after. +func (c *fakeCluster) flapSet(s spell, good, bad []report.Workload) { + have := map[string]report.Workload{} + for _, r := range good { + have[r.ID] = r + } + for _, b := range bad { + g, ok := have[b.ID] + if ok { + g.Since, b.Since = time.Time{}, time.Time{} + if reflect.DeepEqual(g, b) { + continue + } + } + b := cloneWorkload(b) + if !ok { + c.episodes = append(c.episodes, episode{id: b.ID, s: s, bad: func(w *report.Workload, since time.Time) { *w = cloneWorkload(b); w.Since = since }}) + continue + } + c.flap(b.ID, s, func(w *report.Workload, since time.Time) { *w = cloneWorkload(b); w.Since = since }) + } + c.play(time.Now()) +} + +// play puts each flapping record in the form its spell says, and marks +// the ones that changed. +func (c *fakeCluster) play(now time.Time) { + c.mu.Lock() + defer c.mu.Unlock() + for i := range c.episodes { + e := &c.episodes[i] + on := e.s.active(now) + if e.in && on == e.on { + continue + } + e.on, e.in = on, true + if e.good.ID == "" && !on { // there only during the spell + if _, ok := c.recs[e.id]; ok { + delete(c.recs, e.id) + delete(c.changed, e.id) + c.removed = append(c.removed, e.id) + } + continue + } + w := cloneWorkload(e.good) + if on { + e.bad(&w, e.s.began(now)) + } else if !e.good.Since.IsZero() && now.Sub(e.s.began(now)) < e.s.every { + w.Since = e.s.began(now).Add(e.s.on) // right again since the spell ended + } + c.recs[e.id] = w + c.changed[e.id] = true + } +} + +// flapping says whether record id has an episode, so churn leaves it be. +func (c *fakeCluster) flapping(id string) bool { + for _, e := range c.episodes { + if e.id == id { + return true + } + } + return false +} + +func cloneWorkload(w report.Workload) report.Workload { + if w.Facts != nil { + f := make(map[string]any, len(w.Facts)) + for k, v := range w.Facts { + f[k] = v + } + w.Facts = f + } + return w } func newID() string { @@ -69,6 +169,7 @@ func newCluster(name, token string, s clusterShape) *fakeCluster { Since: now.Add(-48 * time.Hour), Facts: map[string]any{"kubelet": kubelet, "os": nodeOS, "cpus": 32, "memory_gib": 256}}) } k := clusterIndex(name) + crasher := "" for i := range s.controllers { ns, owner := fmt.Sprintf("ns-%02d", i%30), fmt.Sprintf("deployment/app-%03d", i) want := s.pods / max(s.controllers, 1) @@ -82,11 +183,8 @@ func newCluster(name, token string, s clusterShape) *fakeCluster { for range want { c.addPod(ns, owner, now) } - if i%97 == 0 { // a few controllers with a crashing pod - id := c.pods[len(c.pods)-1] - p := c.recs[id] - p.State, p.Word, p.Facts = report.StateCrashloop, "Restarting (CrashLoopBackOff)", map[string]any{"restarts": 40, "last_exit": "Error"} - put(p) + if i == 97 && k == 0 { // a pod that crashes for a while after each deploy (scenario.go) + crasher = c.pods[len(c.pods)-1] } } vmGroup := map[string][]string{} @@ -142,6 +240,12 @@ func newCluster(name, token string, s clusterShape) *fakeCluster { if name == "cluster-00" { // the pod the probe changes put(report.Workload{Kind: report.KindPod, ID: "probe", Name: "probe", Namespace: "fleet", Node: "node-00", State: report.StateOK, Word: "Running"}) } + if crasher != "" { + c.flap(crasher, spellCrashloop, func(p *report.Workload, since time.Time) { + p.State, p.Word, p.Since = report.StateCrashloop, "Restarting (CrashLoopBackOff)", since + p.Facts["restarts"], p.Facts["last_exit"], p.Facts["ready"] = 9, "Error", "0/1" + }) + } return c } @@ -212,7 +316,7 @@ func (c *fakeCluster) churn(n int, now time.Time) { i := mrand.N(len(c.pods)) id := c.pods[i] p := c.recs[id] - if p.State == report.StateCrashloop { + if p.State == report.StateCrashloop || c.flapping(id) { continue } if mrand.N(10) == 0 { // replaced: a new pod, the old one gone @@ -372,6 +476,7 @@ func runCluster(ctx context.Context, c *fakeCluster, client *http.Client, pick f } } c.churn(int(c.shape.churn*5), now) + c.play(now) if probed != nil { probeWord, probeAt = c.probe(now), now } diff --git a/scripts/fleet/datalayer.go b/scripts/fleet/datalayer.go index 18207c5..e524a9b 100644 --- a/scripts/fleet/datalayer.go +++ b/scripts/fleet/datalayer.go @@ -20,7 +20,9 @@ import ( // real network; the status pages come in the hosts' reports, as an agent's // fetches do. // Problems on purpose: the leader is fleet-0031, not its usual fleet-0030 -// (a failover nobody moved back), and fleet-0032 is 40 MB behind. +// (a failover nobody moved back), and every so often fleet-0032 falls 40 +// MB behind and HAProxy takes it out of the replicas' pool until it +// catches up (scenario.go, story 2). const patroniPort = 58420 var ( @@ -85,7 +87,9 @@ func servePatroni(ctx context.Context) { case pgHome: role = "sync_standby" case pgLagging: - lag = `,"lag":41943040` + if spellReplicaBehind.active(time.Now()) { + lag = `,"lag":41943040` + } } ms = append(ms, fmt.Sprintf(`{"name":%q,"role":%q,"state":%q,"host":%q,"port":5432,"timeline":7%s}`, hostName(m), role, state, pgAddr(m), lag)) } @@ -131,7 +135,8 @@ func dataFetches(n int, now time.Time) []report.Fetch { fmt.Fprintf(&b, "%s,BACKEND,0,UP,1\n", name) } backend("pg-primary", func(m int) bool { return m == pgLeader }, pgMembers) - backend("pg-replicas", func(m int) bool { return m != pgLeader }, pgMembers) + behind := spellReplicaBehind.active(now) + backend("pg-replicas", func(m int) bool { return m != pgLeader && !(behind && m == pgLagging) }, pgMembers) var apps []int for p := 3; p < 13; p++ { apps = append(apps, p) diff --git a/scripts/fleet/demo.go b/scripts/fleet/demo.go index fb80b57..3f7bd92 100644 --- a/scripts/fleet/demo.go +++ b/scripts/fleet/demo.go @@ -16,17 +16,20 @@ import ( // keep running, and the backups each site keeps, so that every page has // something to show. config writes the files obs-beacond reads; run makes // the hosts report against them. Both work them out from the number of -// agents, so give both the same -n. Problems on purpose: +// agents, so give both the same -n. Problems on purpose, most of them +// coming and going (scenario.go): // -// checks legacy.example.com answers 502 (a warning-only check); -// billing.example.com's certificate ends in 9 days; the -// ping to the last site loses 3% -// units fleet-0041's redis-server has failed -// containers fleet-0007's app-04 exited; fleet-0008's app-09 is stopped -// on purpose -// backups site 1's newest pg-dump is 40 hours old; site 2's shrank to -// a third; site 3's backup disk is 93% full; site 4's -// pg-backup job failed; site 5's md1 is missing a disk +// checks legacy.example.com answers 502 now and then (a warning-only +// check); billing.example.com's certificate ends in 9 days; +// the ping to the last site loses 3%; mx2 lands on a block +// list now and then +// units fleet-0041's redis-server fails now and then, and is +// restarted +// containers fleet-0007's app-04 exits now and then; fleet-0008's app-09 +// is stopped on purpose +// backups site 1's pg-dump is late now and then; site 2's shrinks to a +// third now and then; site 3's backup disk is 93% full; site +// 4's pg-backup job failed; site 5's md1 is missing a disk // role is what a host does in its site, by position (netmap.go). func role(pos int) string { @@ -112,6 +115,9 @@ func demoChecks(hosts int) []demoCheck { out = append(out, demoCheck{id: "web-legacy", name: "legacy.example.com", yaml: "http: https://legacy.example.com/\n severity: warning", from: web, result: func(from int, now time.Time) report.CheckResult { + if !spellLegacy502.active(now) { + return report.CheckResult{ID: "web-legacy", At: now.UTC(), OK: true, Status: 200, Ms: 340} + } return report.CheckResult{ID: "web-legacy", At: now.UTC(), Status: 502, Ms: 2100, Error: "HTTP 502, want 200"} }}, demoCheck{id: "dns-mx", name: "example.com MX", yaml: "dns: example.com\n type: MX\n contains: mx1.example.com", from: web, @@ -126,6 +132,9 @@ func demoChecks(hosts int) []demoCheck { // mx2 landed on a list that flags whole networks: a warning to look at, not a page. demoCheck{id: "rbl-mx2", name: "mx2.example.com on block lists", yaml: "rbl: 192.0.2.26\n severity: warning", from: web[:1], result: func(from int, now time.Time) report.CheckResult { + if !spellBlocklisted.active(now) { + return report.CheckResult{ID: "rbl-mx2", At: now.UTC(), OK: true, Ms: 190, Answer: "clean on 5 of 5"} + } return report.CheckResult{ID: "rbl-mx2", At: now.UTC(), Ms: 210, Answer: "clean on 4 of 5", Error: "listed on dnsbl-1.uceprotect.net (127.0.0.2)"} }}, @@ -204,16 +213,19 @@ func demoScan(n, hosts int, now time.Time) *report.Backups { if name == "system" { size = int64(800+40*k) << 20 } - d := report.BackupDir{Name: name, From: hostName(from), Path: fmt.Sprintf("/srv/backups/%s/%s/%s", hostName(from), name, now.Format("2006-01-02")), - Size: size, PrevSize: size - size/50, Count: 14, Modified: now.Add(-time.Duration(3+k) * time.Hour).UTC()} + d := report.BackupDir{Name: name, From: hostName(from), Size: size, PrevSize: size - size/50, Count: 14, + Modified: now.Add(-time.Duration(3+k) * time.Hour).UTC()} if k == 0 && name == "pg-dump" { - switch site { - case 1: - d.Modified = now.Add(-40 * time.Hour).UTC() - case 2: + switch { + case site == 1 && spellLateBackup.active(now): + // Last night's never came: the newest is the night before's. + d.Modified = spellLateBackup.began(now).Add(-26*time.Hour - 10*time.Minute).UTC() + case site == 2 && spellBackupShrank.active(now): d.PrevSize, d.Size = d.Size, d.Size/3 } } + // Each copy is in a folder named after the day it was made. + d.Path = fmt.Sprintf("/srv/backups/%s/%s/%s", hostName(from), name, d.Modified.Format("2006-01-02")) sc.Received = append(sc.Received, d) } } @@ -328,22 +340,60 @@ func (d *demoHost) setup(r *report.Report) { } for k := range r.Containers.List { c := &r.Containers.List[k] - switch { - case d.n == demoExited && c.Name == "app-04": - c.State, c.Status, c.Health = "exited", "Exited (137) 12 minutes ago", "" - case d.n == demoStoppedOnPurp && c.Name == "app-09": + if d.n == demoStoppedOnPurp && c.Name == "app-09" { c.State, c.Status, c.Health = "exited", "Exited (0) 2 days ago", "" } } } +// containers puts the containers that come and go in their state now. +func (d *demoHost) containers(r *report.Report, now time.Time) { + if d.n != demoExited || r.Containers == nil { + return + } + list := append([]report.Container{}, r.Containers.List...) + for k := range list { + c := &list[k] + if c.Name != "app-04" { + continue + } + if b := spellAppExits.began(now); spellAppExits.active(now) { + c.State, c.Status, c.Health = "exited", fmt.Sprintf("Exited (137) %s ago", fuzzy(now.Sub(b))), "" + } else { + c.State, c.Health = "running", "healthy" + c.Status = fmt.Sprintf("Up %s (healthy)", fuzzy(now.Sub(b.Add(spellAppExits.on)))) + } + } + r.Containers = &report.Containers{List: list} +} + +// fuzzy is a duration as docker ps says it. +func fuzzy(d time.Duration) string { + switch { + case d < time.Minute: + return "Less than a minute" + case d < 2*time.Minute: + return "About a minute" + case d < time.Hour: + return fmt.Sprintf("%d minutes", int(d.Minutes())) + case d < 2*time.Hour: + return "About an hour" + } + return fmt.Sprintf("%d hours", int(d.Hours())) +} + // stamp adds the units, check results and backup scan to a report. func (d *demoHost) stamp(r *report.Report, start, now time.Time) { r.Units = nil for _, u := range demoUnits(d.n) { un := report.Unit{Name: u + ".service", Load: "loaded", Active: "active", Sub: "running", Since: start.UTC()} if d.n == demoFailedUnit && u == "redis-server" { - un.Active, un.Sub, un.Restarts, un.Since = "failed", "failed", 5, time.Time{} + switch b := spellRedisFails.began(now); { + case spellRedisFails.active(now): + un.Active, un.Sub, un.Restarts, un.Since = "failed", "failed", 5, time.Time{} + case b.After(start): // restarted by hand when the spell ended + un.Since = b.Add(spellRedisFails.on).UTC() + } } r.Units = append(r.Units, un) } @@ -351,6 +401,7 @@ func (d *demoHost) stamp(r *report.Report, start, now time.Time) { for _, c := range d.checks { r.Checks = append(r.Checks, c.result(d.n, now)) } + d.containers(r, now) r.Backups = demoScan(d.n, d.hosts, now) if hasDataLayer(d.hosts) { r.Fetches = dataFetches(d.n, now) diff --git a/scripts/fleet/hmc.go b/scripts/fleet/hmc.go index 87d9fcd..c41058b 100644 --- a/scripts/fleet/hmc.go +++ b/scripts/fleet/hmc.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "strings" "time" "github.com/cairnobs/cairnobs/internal/hmc" @@ -18,9 +17,9 @@ const hmcName = "hmc1" // newHMC is three servers: p10-prod runs the four IBM i partitions (their // serial and partition numbers are the ones the partitions say), an AIX -// database and a Linux pair; p10-dr has its attention light on, a VIOS -// the HMC can't reach over RMC and a Linux partition stopped in Open -// Firmware; p9-old is in standby, waiting to be retired. +// database and a Linux pair; p10-dr has its attention light on (a +// serviceable event nobody has closed); p9-old is in standby, waiting to +// be retired. func newHMC(name, token string) *fakeCluster { c := &fakeCluster{name: name, token: token, recs: map[string]report.Workload{}, changed: map[string]bool{}, noUsage: true, source: hmcName} n := 0 @@ -52,9 +51,8 @@ func newHMC(name, token string) *fakeCluster { MachineType: "9105", Model: "22B", Serial: "78DEF45", ProcUnits: 20, ProcFree: 12.25, MemMB: 1 << 20, MemFreeMB: 655_360, AttentionLED: "true", Partitions: []hmc.Partition{vios("p10-dr", 1, 1), vios("p10-dr", 2, 2), part("p10-dr", "aixdb1-dr", "AIX/Linux", "AIX 7.3 7300-03-00-2446", "running", 3, 2, 8, 512), - part("p10-dr", "lnxtest1", "AIX/Linux", "Linux/SUSE 6.4.0-150600.23.25-default 15.6", "open firmware", 4, 0.5, 2, 16), + part("p10-dr", "lnxtest1", "AIX/Linux", "Linux/SUSE 6.4.0-150600.23.25-default 15.6", "running", 4, 0.5, 2, 16), part("p10-dr", "aixtest1", "AIX/Linux", "Unknown", "not activated", 5, 0.5, 1, 16)}} - dr.Partitions[1].RMC = "inactive" old := hmc.Server{UUID: "1f7a9c3e-5b2d-3e84-a6c0-94d2e8b1f733", Name: "p9-old", State: "standby", DetailedState: "None", MachineType: "9009", Model: "42A", Serial: "13XYZ99", ProcUnits: 16, ProcFree: 16, MemMB: 512 << 10, MemFreeMB: 512 << 10, AttentionLED: "false", Partitions: []hmc.Partition{part("p9-old", "aixold1", "AIX/Linux", "AIX 7.2 7200-05-08-2420", "not activated", 1, 2, 4, 64)}} @@ -62,9 +60,6 @@ func newHMC(name, token string) *fakeCluster { now := time.Now() for i, r := range hmc.Records(s) { r.Since = now.Add(-time.Duration(30+i%11) * 24 * time.Hour) - if strings.HasPrefix(r.Word, "Stopped in Open Firmware") || strings.HasPrefix(r.Word, "Running, but the HMC") { - r.Since = now.Add(-time.Duration(2+i%5) * time.Hour) - } c.recs[r.ID] = r } return c diff --git a/scripts/fleet/ibmi.go b/scripts/fleet/ibmi.go index 6dc9bb1..c931288 100644 --- a/scripts/fleet/ibmi.go +++ b/scripts/fleet/ibmi.go @@ -20,12 +20,11 @@ const ibmiHost = 41 // ibmiPartitions are the partitions, in the order -ibmi N takes them. var ibmiPartitions = []string{"prod1", "prod2", "ha1", "dev1"} -// newIBMi is the k'th partition: prod1 busy, its system ASP past IBM i's -// threshold, a disk unit's parity being rebuilt and the nightly batch -// stuck on a question; prod2 with ORDERS ended, its job queue filling, -// and its backup interface down; ha1 the PowerHA standby, its -// independent ASP varied off; dev1 on 7.4 without the authority to read -// its Technology Refresh. +// newIBMi is the k'th partition: prod1 busy, its nightly batch stuck on a +// question now and then; prod2's ORDERS subsystem ended for a while now +// and then, its job queue filling meanwhile (scenario.go); ha1 the +// PowerHA standby, its independent ASP varied off; dev1 on 7.4 without +// the authority to read its Technology Refresh. func newIBMi(name, token string, k int) *fakeCluster { part := ibmiPartitions[k] c := &fakeCluster{name: name, token: token, recs: map[string]report.Workload{}, changed: map[string]bool{}, noUsage: true, source: part} @@ -39,49 +38,62 @@ func newIBMi(name, token string, k int) *fakeCluster { pct := min(100, base*(0.6+0.4*day)*(0.9+0.2*rand.Float64())) return []report.Usage{{ID: report.ClusterSelf, CPU: math.Round(pct*units) / 100, CPUOf: units}} } - sysUsed := []float64{91.4, 72.8, 64.1, 48.0}[k] - s := &ibmi.Snapshot{Name: part, Release: "7.5", TR: 6, At: time.Now(), - System: ibmi.System{HostName: "S" + part, PartitionName: strings.ToUpper(part), PartitionID: k + 2, Partitions: 6, MachineType: "9105", MachineModel: "42A", - Serial: "78ABC12", CPUs: int(units * 2), Capacity: units, MainStorageKB: int64(units*64) << 20, SystemASPMB: 2_400_000, - SystemASPUsed: sysUsed, TempMB: 38_000, Jobs: []int{18204, 14550, 6120, 3307}[k], MaxJobs: 163520, - ActiveJobs: []int{612, 488, 201, 97}[k], Threads: []int{2870, 2210, 890, 402}[k]}, - ASPs: []ibmi.ASP{{Number: 1, State: "NONE", Type: "SYSTEM", Units: 8, Present: "ALL", TotalMB: 2_400_000, - FreeMB: int64(2_400_000 * (100 - sysUsed) / 100), Threshold: 90}}} - switch part { - case "prod1", "ha1": - ia := ibmi.ASP{Number: 33, Name: "PRODIASP", State: "AVAILABLE", Type: "PRIMARY", Units: 6, Present: "ALL", TotalMB: 3_600_000, FreeMB: 1_440_000, Threshold: 90} - if part == "ha1" { - ia.State = "VARIED OFF" + sysUsed := []float64{84.6, 72.8, 64.1, 48.0}[k] + snap := func(bad bool) *ibmi.Snapshot { + s := &ibmi.Snapshot{Name: part, Release: "7.5", TR: 6, At: time.Now(), + System: ibmi.System{HostName: "S" + part, PartitionName: strings.ToUpper(part), PartitionID: k + 2, Partitions: 6, MachineType: "9105", MachineModel: "42A", + Serial: "78ABC12", CPUs: int(units * 2), Capacity: units, MainStorageKB: int64(units*64) << 20, SystemASPMB: 2_400_000, + SystemASPUsed: sysUsed, TempMB: 38_000, Jobs: []int{18204, 14550, 6120, 3307}[k], MaxJobs: 163520, + ActiveJobs: []int{612, 488, 201, 97}[k], Threads: []int{2870, 2210, 890, 402}[k]}, + ASPs: []ibmi.ASP{{Number: 1, State: "NONE", Type: "SYSTEM", Units: 8, Present: "ALL", TotalMB: 2_400_000, + FreeMB: int64(2_400_000 * (100 - sysUsed) / 100), Threshold: 90}}} + switch part { + case "prod1", "ha1": + ia := ibmi.ASP{Number: 33, Name: "PRODIASP", State: "AVAILABLE", Type: "PRIMARY", Units: 6, Present: "ALL", TotalMB: 3_600_000, FreeMB: 1_440_000, Threshold: 90} + if part == "ha1" { + ia.State = "VARIED OFF" + } + s.ASPs = append(s.ASPs, ia) + case "dev1": + s.Release, s.TR = "7.4", 0 } - s.ASPs = append(s.ASPs, ia) - case "dev1": - s.Release, s.TR = "7.4", 0 - } - sbs := func(lib, name, status string, jobs int) ibmi.Subsystem { - return ibmi.Subsystem{Library: lib, Name: name, Status: status, Jobs: jobs, Controlling: name == "QCTL"} - } - s.Subsys = []ibmi.Subsystem{sbs("QSYS", "QCTL", "ACTIVE", 3), sbs("QSYS", "QINTER", "ACTIVE", 20+k*7), sbs("QSYS", "QBATCH", "ACTIVE", 6+k), - sbs("QSYS", "QUSRWRK", "ACTIVE", 60+k*9), sbs("QSYS", "QSERVER", "ACTIVE", 14), sbs("QSYS", "QSYSWRK", "ACTIVE", 110), sbs("QSYS", "QSPL", "ACTIVE", 4), - sbs("QSYS", "QHTTPSVR", "ACTIVE", 9), sbs("QSYS", "QSNADS", "INACTIVE", 0), sbs("QSYS", "QCMN", "ACTIVE", 2), sbs("ORDLIB", "ORDERS", "ACTIVE", 12)} - s.Ifaces = []ibmi.Interface{{Address: fmt.Sprintf("10.0.4.%d", 20+k), Prefix: 24, Line: "ETHLINE", Status: "ACTIVE", Autostart: true}, - {Address: fmt.Sprintf("10.0.9.%d", 20+k), Prefix: 24, Line: "ETHBKP", Status: "ACTIVE", Autostart: true}} - switch part { - case "prod1": - s.Disks = []ibmi.Disk{{ASP: 1, Unit: 6, Resource: "DMP006", Hardware: "PARITY REBUILD"}} - s.Inquiries = []ibmi.Message{{Key: "0001A2F0", ID: "CPA0701", Inquiry: true, Severity: 99, Job: "418276/BATCH/NIGHTLY", User: "BATCH", - Text: "MCH1211 received by procedure ORDPOST. (C D I R)", Age: 47 * time.Minute}} - s.Unanswered = 1 - case "prod2": - s.Subsys[len(s.Subsys)-1].Status, s.Subsys[len(s.Subsys)-1].Jobs = "INACTIVE", 0 // ORDERS: ended, not started again - s.Ifaces[1].Status = "INACTIVE" // its backup line, set to start - s.JobQueues = []ibmi.JobQueue{{Library: "ORDLIB", Name: "ORDERSQ", Waiting: 14}} // ORDERS isn't there to take them - case "ha1": - s.Subsys = s.Subsys[:len(s.Subsys)-1] // ORDERS runs on the active node only + sbs := func(lib, name, status string, jobs int) ibmi.Subsystem { + return ibmi.Subsystem{Library: lib, Name: name, Status: status, Jobs: jobs, Controlling: name == "QCTL"} + } + s.Subsys = []ibmi.Subsystem{sbs("QSYS", "QCTL", "ACTIVE", 3), sbs("QSYS", "QINTER", "ACTIVE", 20+k*7), sbs("QSYS", "QBATCH", "ACTIVE", 6+k), + sbs("QSYS", "QUSRWRK", "ACTIVE", 60+k*9), sbs("QSYS", "QSERVER", "ACTIVE", 14), sbs("QSYS", "QSYSWRK", "ACTIVE", 110), sbs("QSYS", "QSPL", "ACTIVE", 4), + sbs("QSYS", "QHTTPSVR", "ACTIVE", 9), sbs("QSYS", "QSNADS", "INACTIVE", 0), sbs("QSYS", "QCMN", "ACTIVE", 2), sbs("ORDLIB", "ORDERS", "ACTIVE", 12)} + s.Ifaces = []ibmi.Interface{{Address: fmt.Sprintf("10.0.4.%d", 20+k), Prefix: 24, Line: "ETHLINE", Status: "ACTIVE", Autostart: true}, + {Address: fmt.Sprintf("10.0.9.%d", 20+k), Prefix: 24, Line: "ETHBKP", Status: "ACTIVE", Autostart: true}} + switch { + case part == "prod1" && bad: + s.Inquiries = []ibmi.Message{{Key: "0001A2F0", ID: "CPA0701", Inquiry: true, Severity: 99, Job: "418276/BATCH/NIGHTLY", User: "BATCH", + Text: "MCH1211 received by procedure ORDPOST. (C D I R)", Age: 7 * time.Minute}} + s.Unanswered = 1 + case part == "prod2" && bad: + s.Subsys[len(s.Subsys)-1].Status, s.Subsys[len(s.Subsys)-1].Jobs = "INACTIVE", 0 // ORDERS: ended, not started again + s.JobQueues = []ibmi.JobQueue{{Library: "ORDLIB", Name: "ORDERSQ", Waiting: 14}} // ORDERS isn't there to take them + case part == "ha1": + s.Subsys = s.Subsys[:len(s.Subsys)-1] // ORDERS runs on the active node only + } + return s } now := time.Now() - for _, r := range ibmi.Records(s) { - r.Since = now.Add(-time.Duration(40+k*7) * 24 * time.Hour) + recs := func(bad bool) []report.Workload { + out := ibmi.Records(snap(bad)) + for i := range out { + out[i].Since = now.Add(-time.Duration(40+k*7) * 24 * time.Hour) + } + return out + } + for _, r := range recs(false) { c.recs[r.ID] = r } + switch part { + case "prod1": + c.flapSet(spellInquiry, recs(false), recs(true)) + case "prod2": + c.flapSet(spellOrdersDown, recs(false), recs(true)) + } return c } diff --git a/scripts/fleet/main.go b/scripts/fleet/main.go index 5f47351..0e18ad9 100644 --- a/scripts/fleet/main.go +++ b/scripts/fleet/main.go @@ -589,8 +589,8 @@ const ( var vmRoles = [libvirtVMs]string{"db", "web", "cache", "build", "mq", "ldap", "backup", "monitor", "runner", "jump", "proxy", "spare"} -// hypervisorVMs are host i's domains: all running, but its spare is shut -// off, and on the third hypervisor one has crashed. +// hypervisorVMs are host i's domains: all running but its spare, which is +// shut off. func hypervisorVMs(i int, now time.Time) []report.Workload { out := make([]report.Workload, libvirtVMs) for k := range out { @@ -598,11 +598,8 @@ func hypervisorVMs(i int, now time.Time) []report.Workload { r := report.Workload{Kind: report.KindVM, ID: id, Name: fmt.Sprintf("%s-%02d", vmRoles[k], i/hypervisorEvery), State: report.StateOK, Word: "Running", Since: now.Add(-time.Duration(3+k) * 24 * time.Hour).Truncate(time.Second), Facts: map[string]any{"firmware_uuid": id, "autostart": k != libvirtVMs-1, "hypervisor": "libvirt", "vcpus": 2 << (k % 3), "memory_gib": float64(int(4) << (k % 3))}} - switch { - case k == libvirtVMs-1: + if k == libvirtVMs-1 { r.State, r.Word = report.StateStopped, "Shut off" - case i/hypervisorEvery == 2 && k == 7: - r.State, r.Word, r.Since = report.StateDown, "Crashed", now.Add(-47*time.Minute).Truncate(time.Second) } out[k] = r } diff --git a/scripts/fleet/nutanix.go b/scripts/fleet/nutanix.go index 5f4eea7..a8d6e8f 100644 --- a/scripts/fleet/nutanix.go +++ b/scripts/fleet/nutanix.go @@ -35,21 +35,12 @@ func newNutanix(name, token string, vms int) *fakeCluster { r := report.Workload{Kind: report.KindNode, ID: fmt.Sprintf("a1b2c3d4-%04d-4000-8000-%012d", cl, n), Name: host(cl, n), State: report.StateOK, Word: "Connected", Since: now.Add(-time.Duration(20+n) * 24 * time.Hour), Facts: map[string]any{"cluster": peName(cl), "cpus": 64, "memory_gib": 1024.0, "version": "AHV 10.0.1", "model": "NX-8170-G9"}} - switch { - case cl == 1 && n == 4: - r.State, r.Word, r.Since = report.StateUpdating, "In maintenance mode", now.Add(-20*time.Minute) - case cl == 2 && n == 2: - r.State, r.Word, r.Facts["alerts"] = report.StateDegraded, "Alert from Prism: Disk Failure", "Disk Failure (critical)" - } put(r) } for k, ct := range []string{"default-container", "vdi-images", "db-data"} { r := report.Workload{Kind: report.KindPVC, ID: fmt.Sprintf("c0ffee00-%04d-4000-8000-%012d", cl, k), Name: ct, State: report.StateOK, Word: "Available", Since: now.Add(-200 * 24 * time.Hour), Facts: map[string]any{"type": "storage container", "cluster": peName(cl), "capacity_gib": 81920.0, "replication_factor": 2}} - if cl == 0 && k == 2 { - r.State, r.Word, r.Facts["alerts"] = report.StateDegraded, "Alert from Prism: Storage Container Space Usage Exceeded", "Storage Container Space Usage Exceeded (critical)" - } put(r) } } @@ -72,5 +63,8 @@ func newNutanix(name, token string, vms int) *fakeCluster { } put(r) } + c.flap(fmt.Sprintf("a1b2c3d4-%04d-4000-8000-%012d", 1, 4), spellAHVUpdate, func(r *report.Workload, since time.Time) { // an AOS upgrade's rolling restart (scenario.go) + r.State, r.Word, r.Since = report.StateUpdating, "In maintenance mode", since + }) return c } diff --git a/scripts/fleet/proxmox.go b/scripts/fleet/proxmox.go index 22856ca..b24d4d4 100644 --- a/scripts/fleet/proxmox.go +++ b/scripts/fleet/proxmox.go @@ -45,9 +45,6 @@ func newProxmox(name, token string, guests int) *fakeCluster { switch { case k%23 == 22: r.State, r.Word, r.Facts["ha"] = report.StateStopped, "Stopped", "stopped" - case k == 41: - r.State, r.Word, r.Facts["ha"] = report.StateDown, "HA can't run it (error)", "error" - r.Since = now.Add(-18 * time.Minute) case k == 57: r.State, r.Word, r.Facts["lock"] = report.StateUpdating, "Migrating", "migrate" case k%50 == 9: @@ -55,5 +52,8 @@ func newProxmox(name, token string, guests int) *fakeCluster { } put(r) } + c.flap("lxc/141", spellPVEHA, func(r *report.Workload, since time.Time) { // scenario.go + r.State, r.Word, r.Since, r.Facts["ha"] = report.StateDown, "HA can't run it (error)", since, "error" + }) return c } diff --git a/scripts/fleet/pure.go b/scripts/fleet/pure.go index 65c3885..776e8ac 100644 --- a/scripts/fleet/pure.go +++ b/scripts/fleet/pure.go @@ -27,25 +27,21 @@ func newPure(name, token string, k int) *fakeCluster { } // newFlashBlade is fb-prod-01 (docs/pure.md, step 5): ten blades, one -// evacuating; a failed fan; 400 file systems, cluster-02's first 100 VM -// disks among them (Portworx names them after their PersistentVolumes), -// one whose replication to fb-dr-01 is unhealthy; 40 buckets. +// evacuating now and then (scenario.go); 400 file systems, cluster-02's +// first 100 VM disks among them (Portworx names them after their +// PersistentVolumes); 40 buckets. func newFlashBlade(name, token string, k int) *fakeCluster { arr := pureArrays[k] c := &fakeCluster{name: name, token: token, recs: map[string]report.Workload{}, changed: map[string]bool{}, noUsage: true, source: arr} c.usageFn = func(now time.Time) []report.Usage { return pureIO(k, now) } now := time.Now() put := func(w report.Workload) { c.recs[w.ID] = w } - put(report.Workload{Kind: report.KindCluster, ID: report.ClusterSelf, Name: arr, State: report.StateDegraded, Word: "Hardware: CH1.FAN2 critical (Fan speed is 0 RPM)", - Since: now.Add(-95 * time.Minute), Facts: map[string]any{"distribution": "Purity//FB", "version": "4.5.6", "model": "FlashBlade//S", "capacity_tib": 2000.0, - "used_pct": 61.0, "data_reduction": 3.1, "hardware": "CH1.FAN2 critical (Fan speed is 0 RPM)", "alerts": "CH1.FAN2: Fan failure (critical)"}}) + put(report.Workload{Kind: report.KindCluster, ID: report.ClusterSelf, Name: arr, State: report.StateOK, Word: "API answering", + Since: now.Add(-200 * 24 * time.Hour), Facts: map[string]any{"distribution": "Purity//FB", "version": "4.5.6", "model": "FlashBlade//S", "capacity_tib": 2000.0, + "used_pct": 61.0, "data_reduction": 3.1}}) for i := 1; i <= 10; i++ { - r := report.Workload{Kind: report.KindNode, ID: fmt.Sprintf("ch1.fb%d", i), Name: fmt.Sprintf("CH1.FB%d", i), State: report.StateOK, Word: "Healthy", - Since: now.Add(-200 * 24 * time.Hour), Facts: map[string]any{"raw_tib": 48.0}} - if i == 6 { - r.State, r.Word, r.Since = report.StateUpdating, "Evacuating its data (37%)", now.Add(-2*time.Hour) - } - put(r) + put(report.Workload{Kind: report.KindNode, ID: fmt.Sprintf("ch1.fb%d", i), Name: fmt.Sprintf("CH1.FB%d", i), State: report.StateOK, Word: "Healthy", + Since: now.Add(-200 * 24 * time.Hour), Facts: map[string]any{"raw_tib": 48.0}}) } put(report.Workload{Kind: report.KindCluster, ID: "connection/c-fb-dr-01", Name: "connection/fb-dr-01", State: report.StateOK, Word: "Connected", Since: now.Add(-90 * 24 * time.Hour), Facts: map[string]any{"type": "async-replication", "version": "4.5.6"}}) @@ -62,9 +58,7 @@ func newFlashBlade(name, token string, k int) *fakeCluster { r.Name, r.Word, r.Facts["protocols"] = "home", "Shared over NFSv3, NFSv4.1, SMB", "NFSv3, NFSv4.1, SMB" r.Facts["replication"] = "to fb-dr-01, replicating, 4 min behind" case n == 101: - r.Name, r.State = "scratch", report.StateDegraded - r.Word = "Replication to fb-dr-01 is unhealthy: Target file system is full" - r.Facts["replication"] = "to fb-dr-01, unhealthy, 5 h behind" + r.Name = "scratch" case n%50 == 7: r.Word = "Not shared over any protocol" delete(r.Facts, "protocols") @@ -80,6 +74,10 @@ func newFlashBlade(name, token string, k int) *fakeCluster { } put(r) } + c.flap("ch1.fb6", spellBladeEvac, func(r *report.Workload, since time.Time) { + done := min(95, 5+int(time.Since(since).Minutes())) + r.State, r.Word, r.Since = report.StateUpdating, fmt.Sprintf("Evacuating its data (%d%%)", done), since + }) return c } @@ -99,12 +97,8 @@ func newFlashArray(name, token string, k, volumes int) *fakeCluster { } put(self) for n, mode := range []string{"primary", "secondary"} { - r := report.Workload{Kind: report.KindNode, ID: fmt.Sprintf("ct%d", n), Name: fmt.Sprintf("CT%d", n), State: report.StateOK, Word: "Ready, " + mode, - Since: now.Add(-40 * 24 * time.Hour), Facts: map[string]any{"mode": mode, "model": "FA-X70R4", "version": "6.7.3"}} - if arr == "fa-vdi-01" && n == 1 { - r.State, r.Word, r.Since = report.StateUpdating, "Updating", now.Add(-12*time.Minute) - } - put(r) + put(report.Workload{Kind: report.KindNode, ID: fmt.Sprintf("ct%d", n), Name: fmt.Sprintf("CT%d", n), State: report.StateOK, Word: "Ready, " + mode, + Since: now.Add(-40 * 24 * time.Hour), Facts: map[string]any{"mode": mode, "model": "FA-X70R4", "version": "6.7.3"}}) } switch arr { case "fa-prod-01", "fa-prod-02": @@ -116,8 +110,8 @@ func newFlashArray(name, token string, k, volumes int) *fakeCluster { put(report.Workload{Kind: report.KindCluster, ID: "connection/c-fa-dr-01", Name: "connection/fa-dr-01", State: report.StateOK, Word: "Connected", Since: now.Add(-60 * 24 * time.Hour), Facts: map[string]any{"type": "async-replication", "version": "6.7.3"}}) case "fa-dr-01": - put(report.Workload{Kind: report.KindCluster, ID: "connection/c-fa-prod-01", Name: "connection/fa-prod-01", State: report.StateDegraded, - Word: "Connected, but its paths aren't redundant", Since: now.Add(-50 * time.Minute), Facts: map[string]any{"type": "async-replication", "version": "6.7.3"}}) + put(report.Workload{Kind: report.KindCluster, ID: "connection/c-fa-prod-01", Name: "connection/fa-prod-01", State: report.StateOK, + Word: "Connected", Since: now.Add(-60 * 24 * time.Hour), Facts: map[string]any{"type": "async-replication", "version": "6.7.3"}}) } prefix := map[string]string{"fa-prod-01": "k8s-pvc", "fa-prod-02": "k8s-pvc", "fa-dr-01": "dr-copy", "fa-vdi-01": "vdi"}[arr] for n := range volumes { @@ -146,6 +140,16 @@ func newFlashArray(name, token string, k, volumes int) *fakeCluster { } put(r) } + switch arr { + case "fa-vdi-01": + c.flap("ct1", spellVDIUpdate, func(r *report.Workload, since time.Time) { + r.State, r.Word, r.Since = report.StateUpdating, "Updating", since + }) + case "fa-dr-01": + c.flap("connection/c-fa-prod-01", spellDRPaths, func(r *report.Workload, since time.Time) { + r.State, r.Word, r.Since = report.StateDegraded, "Connected, but its paths aren't redundant", since + }) + } return c } diff --git a/scripts/fleet/scenario.go b/scripts/fleet/scenario.go new file mode 100644 index 0000000..a920679 --- /dev/null +++ b/scripts/fleet/scenario.go @@ -0,0 +1,91 @@ +package main + +import "time" + +// What goes wrong in the demo, and when (#35). A real fleet is mostly +// fine: a few things stay wrong until someone gets to them, and the rest +// come and go. So the demo has +// +// - three stories, each a cause with its effects: +// 1. fa-prod-01 lost a drive: the array is degraded and its writes are +// slow while the drive's data is rebuilt (all the time); +// 2. checkout errors: every 2 hours fleet-0032, a PostgreSQL replica, +// falls behind, HAProxy takes it out, and the shop's checkout answers +// with errors until it catches up (20 minutes); +// 3. the legacy datacenter: every 4 hours its server stops answering +// Akamai's liveness test, GTM has nowhere to send the legacy property, +// and its CP code fails (30 minutes); +// - a few lasting warnings anyone would have: a failover nobody moved +// back, a weak key, a VM snapshot left behind, a datastore filling, a +// zone on Flexible SSL, a renewal waiting on someone; +// - problems that come and go: a unit that fails and is restarted, a +// container that exits, a check that flaps, a host update that takes +// a while, a late backup, an IBM i job waiting on an answer. +// +// Spells count from the Unix epoch, not from when the simulator started, +// so every part of it agrees on what's happening now, a restarted demo +// picks up where the clock is, and the history the demo starts with can +// be worked out for any time in the past. + +// spell is when one problem is happening: for on, once every every, at +// into each period. A zero every means all the time. +type spell struct { + every, at, on time.Duration +} + +// always is a problem that's there all the time. +var always = spell{} + +func (s spell) active(t time.Time) bool { + if s.every == 0 { + return true + } + p := time.Duration(t.UnixNano()) % s.every + return p >= s.at && p < s.at+s.on +} + +// began is when the current spell started, or the last one if it's over; +// a long time ago for a problem that's always there. +func (s spell) began(t time.Time) time.Time { + if s.every == 0 { + return t.Add(-9 * 24 * time.Hour) + } + p := time.Duration(t.UnixNano()) % s.every + start := t.Add(-p + s.at) + if start.After(t) { + start = start.Add(-s.every) + } + return start +} + +// The spells, by story. Periods aren't multiples of each other, so the +// mix on screen changes through the day. +var ( + // Story 2: checkout errors. + spellReplicaBehind = spell{every: 2 * time.Hour, at: 40 * time.Minute, on: 20 * time.Minute} + // Story 3: the legacy datacenter. + spellLegacyDC = spell{every: 4 * time.Hour, at: 70 * time.Minute, on: 30 * time.Minute} + + // Coming and going. + spellRedisFails = spell{every: 110 * time.Minute, at: 20 * time.Minute, on: 12 * time.Minute} + spellAppExits = spell{every: 75 * time.Minute, at: 5 * time.Minute, on: 8 * time.Minute} + spellLegacy502 = spell{every: 140 * time.Minute, at: 30 * time.Minute, on: 15 * time.Minute} + spellLBFlaps = spell{every: 100 * time.Minute, at: 60 * time.Minute, on: 15 * time.Minute} + spellCrashloop = spell{every: 150 * time.Minute, at: 15 * time.Minute, on: 20 * time.Minute} + spellESXiUpdate = spell{every: 4 * time.Hour, at: 10 * time.Minute, on: 40 * time.Minute} + spellESXiMemory = spell{every: 130 * time.Minute, at: 50 * time.Minute, on: 15 * time.Minute} + spellAHVUpdate = spell{every: 270 * time.Minute, at: 100 * time.Minute, on: 40 * time.Minute} + spellPVEHA = spell{every: 150 * time.Minute, at: 90 * time.Minute, on: 20 * time.Minute} + spellDRPaths = spell{every: 3 * time.Hour, at: 130 * time.Minute, on: 30 * time.Minute} + spellBladeEvac = spell{every: 6 * time.Hour, at: 2 * time.Hour, on: 60 * time.Minute} + spellVDIUpdate = spell{every: 5 * time.Hour, at: 4 * time.Hour, on: 45 * time.Minute} + spellInquiry = spell{every: 3 * time.Hour, at: 110 * time.Minute, on: 30 * time.Minute} + spellOrdersDown = spell{every: 200 * time.Minute, at: 80 * time.Minute, on: 25 * time.Minute} + spellVaultEdgeDown = spell{every: 3 * time.Hour, at: 150 * time.Minute, on: 20 * time.Minute} + spellBlocklisted = spell{every: 5 * time.Hour, at: 3 * time.Hour, on: 40 * time.Minute} + spellLateBackup = spell{every: 6 * time.Hour, at: 5 * time.Hour, on: 50 * time.Minute} + spellBackupShrank = spell{every: 6 * time.Hour, at: 2 * time.Hour, on: 50 * time.Minute} + spellSQLAgent = spell{every: 4 * time.Hour, at: 3 * time.Hour, on: 45 * time.Minute} + spellFSCrash = spell{every: 8 * time.Hour, at: 7 * time.Hour, on: time.Hour} + spellActivation = spell{every: 3 * time.Hour, at: 20 * time.Minute, on: 80 * time.Minute} +) diff --git a/scripts/fleet/scenario_test.go b/scripts/fleet/scenario_test.go new file mode 100644 index 0000000..dbbc14f --- /dev/null +++ b/scripts/fleet/scenario_test.go @@ -0,0 +1,106 @@ +package main + +import ( + "testing" + "time" + + "github.com/cairnobs/cairnobs/internal/report" +) + +func TestSpell(t *testing.T) { + s := spell{every: time.Hour, at: 20 * time.Minute, on: 10 * time.Minute} + day := time.Date(2026, 11, 16, 0, 0, 0, 0, time.UTC) + for _, c := range []struct { + at time.Duration + active bool + began time.Duration + }{ + {19 * time.Minute, false, -40 * time.Minute}, + {20 * time.Minute, true, 20 * time.Minute}, + {29 * time.Minute, true, 20 * time.Minute}, + {30 * time.Minute, false, 20 * time.Minute}, + {80 * time.Minute, true, 80 * time.Minute}, + } { + now := day.Add(c.at) + if s.active(now) != c.active || !s.began(now).Equal(day.Add(c.began)) { + t.Errorf("at %s: active %v began %s", c.at, s.active(now), s.began(now).Sub(day)) + } + } + if !always.active(day) || !always.began(day).Before(day) { + t.Error("always") + } +} + +// The demo stays calm: across a day, no more than a few of the problems +// that come and go are on at once, and every one of them is on some of +// the time and off the rest. +func TestScenarioIsCalm(t *testing.T) { + spells := map[string]spell{"replica": spellReplicaBehind, "legacy dc": spellLegacyDC, "redis": spellRedisFails, "app": spellAppExits, + "502": spellLegacy502, "lb": spellLBFlaps, "crashloop": spellCrashloop, "esxi update": spellESXiUpdate, "esxi memory": spellESXiMemory, + "ahv": spellAHVUpdate, "pve": spellPVEHA, "dr paths": spellDRPaths, "blade": spellBladeEvac, "vdi": spellVDIUpdate, + "inquiry": spellInquiry, "orders": spellOrdersDown, "vault edge": spellVaultEdgeDown, "blocklist": spellBlocklisted, + "late backup": spellLateBackup, "shrank": spellBackupShrank, "sql agent": spellSQLAgent, "fs crash": spellFSCrash, "activation": spellActivation} + on := map[string]int{} + most, total, steps := 0, 0, 0 + day := time.Date(2026, 11, 16, 0, 0, 0, 0, time.UTC) + for m := 0; m < 24*60; m++ { + now := day.Add(time.Duration(m) * time.Minute) + n := 0 + for name, s := range spells { + if s.on <= 0 || s.at+s.on > s.every { + t.Fatalf("%s: on %s at %s of %s", name, s.on, s.at, s.every) + } + if s.active(now) { + on[name]++ + n++ + } + } + most, total, steps = max(most, n), total+n, steps+1 + } + for name := range spells { + if on[name] == 0 || on[name] == 24*60 { + t.Errorf("%s is on %d minutes of the day", name, on[name]) + } + } + if avg := float64(total) / float64(steps); avg > 4 || most > 9 { + t.Errorf("on at once: %.1f on average, %d at most", avg, most) + } +} + +// A flapping record goes wrong during its spell and comes right after; a +// record only the bad snapshot has comes and goes. +func TestFlapSet(t *testing.T) { + c := &fakeCluster{recs: map[string]report.Workload{}, changed: map[string]bool{}} + good := []report.Workload{{ID: "a", State: report.StateOK, Word: "Active"}, {ID: "b", State: report.StateOK, Word: "Fine"}} + bad := []report.Workload{{ID: "a", State: report.StateDown, Word: "Inactive"}, {ID: "b", State: report.StateOK, Word: "Fine"}, + {ID: "q", State: report.StateDegraded, Word: "14 jobs waiting"}} + for _, r := range good { + c.recs[r.ID] = r + } + s := spell{every: time.Hour, at: 0, on: 30 * time.Minute} + c.flapSet(s, good, bad) + if len(c.episodes) != 2 { + t.Fatalf("%d episodes, want a and q", len(c.episodes)) + } + hour := time.Date(2026, 11, 16, 3, 0, 0, 0, time.UTC) + c.play(hour.Add(5 * time.Minute)) + if c.recs["a"].State != report.StateDown || c.recs["q"].ID != "q" { + t.Fatalf("in the spell: %+v %+v", c.recs["a"], c.recs["q"]) + } + c.removed = nil + c.play(hour.Add(40 * time.Minute)) + if _, there := c.recs["q"]; c.recs["a"].State != report.StateOK || there || len(c.removed) != 1 { + t.Fatalf("after: %+v, q there %v, removed %v", c.recs["a"], there, c.removed) + } +} + +// IBM i's flapping is only what the bad snapshot changes, not the whole +// partition. +func TestIBMiFlapsLittle(t *testing.T) { + for k, part := range ibmiPartitions[:2] { + c := newIBMi("fleet-0041", "t", k) + if n := len(c.episodes); n == 0 || n > 4 { + t.Errorf("%s: %d records flap", part, n) + } + } +} diff --git a/scripts/fleet/vault.go b/scripts/fleet/vault.go index f9da903..9c47ebd 100644 --- a/scripts/fleet/vault.go +++ b/scripts/fleet/vault.go @@ -31,9 +31,11 @@ import ( // - vault-prod: Vault Enterprise, three healthy Raft servers, DR // replication streaming to its secondary, a license 300 days out, an // audit device, and a PKI mount (pki_int) with 400 services' -// certificates (some ending soon) and 2,000 short-lived ones. -// - vault-edge: OpenBao, three Raft servers with one failed (no failure -// tolerance left), and an intermediate CA 12 days from its end. +// certificates (one ending within a week, its renewal overdue) and +// 2,000 short-lived ones. +// - vault-edge: OpenBao, three Raft servers, one of which fails now and +// then and leaves no failure tolerance (scenario.go), and an +// intermediate CA 12 days from its end. // // They listen on vaultPort and the port after it; -vault's directory // keeps their certificate and key, so vault.yaml (written there) stays @@ -74,7 +76,7 @@ func serveVaults(ctx context.Context, dir string) { newFakeVault("vault-prod", "1.21.4+ent", true, "pki_int", now, 300*24*time.Hour, 400, 2000, []fakeRaft{{"v1", "leader", "alive"}, {"v2", "voter", "alive"}, {"v3", "voter", "alive"}}, 1, 18_000), newFakeVault("vault-edge", "2.4.1", false, "pki_edge", now, 12*24*time.Hour, 40, 0, - []fakeRaft{{"e1", "leader", "alive"}, {"e2", "voter", "alive"}, {"e3", "voter", "failed"}}, 0, 900), + []fakeRaft{{"e1", "leader", "alive"}, {"e2", "voter", "alive"}, {"e3", "voter", "alive"}}, 1, 900), } var yaml strings.Builder yaml.WriteString("vault:\n") @@ -153,14 +155,11 @@ func newFakeVault(name, version string, ent bool, mount string, now time.Time, c day := 24 * time.Hour for i := range services { cn := fmt.Sprintf("svc-%03d.%s.internal", i, strings.TrimPrefix(name, "vault-")) - // Most a long way from their end; a few within the warning's 21 - // days, fewer within the critical 7. + // Most a long way from their end; one of vault-prod's within a + // week, its renewal overdue. left := time.Duration(25+(i*37)%340) * day - switch { - case i%97 == 5: - left = 5 * day - case i%41 == 7: - left = 15 * day + if name == "vault-prod" && i == 102 { + left = 6 * day } for k, end := range []time.Duration{left - 30*day, left} { // the one it replaced, then the renewal x, p := sign(&x509.Certificate{Subject: pkix.Name{CommonName: cn}, DNSNames: []string{cn}, NotBefore: now.Add(end - 365*day), NotAfter: now.Add(end)}, @@ -209,8 +208,11 @@ func (v *fakeVault) ServeHTTP(w http.ResponseWriter, r *http.Request) { case p == "sys/leader": out = map[string]any{"ha_enabled": true, "is_self": true, "leader_address": "https://" + v.servers[0].id + "." + v.name + ":8200"} case p == "sys/storage/raft/autopilot/state": - servers, voters, healthy := map[string]any{}, []string{}, true + servers, voters, healthy, ft := map[string]any{}, []string{}, true, v.ft for _, s := range v.servers { + if v.name == "vault-edge" && s.id == "e3" && spellVaultEdgeDown.active(now) { + s.node, s.status, ft = "failed", "voter", 0 + } ok := s.node == "alive" healthy = healthy && ok sv := map[string]any{"id": s.id, "name": v.name + "-" + s.id, "address": s.id + ":8201", "node_status": s.node, "healthy": ok, @@ -221,7 +223,7 @@ func (v *fakeVault) ServeHTTP(w http.ResponseWriter, r *http.Request) { servers[s.id] = sv voters = append(voters, s.id) } - out = data(map[string]any{"healthy": healthy, "failure_tolerance": v.ft, "voters": voters, "servers": servers}) + out = data(map[string]any{"healthy": healthy, "failure_tolerance": ft, "voters": voters, "servers": servers}) case p == "auth/token/lookup-self": out = data(map[string]any{"display_name": "token-obs-beacond", "policies": []string{"default", "obs-monitor"}, "ttl": 20 * 3600, "creation_ttl": 24 * 3600, "renewable": true, "type": "service"}) diff --git a/scripts/fleet/vsphere.go b/scripts/fleet/vsphere.go index b256683..055f3a3 100644 --- a/scripts/fleet/vsphere.go +++ b/scripts/fleet/vsphere.go @@ -30,12 +30,6 @@ func newVSphere(name, token string, vms int) *fakeCluster { r := report.Workload{Kind: report.KindNode, ID: fmt.Sprintf("host-%d", 1000+n), Name: fmt.Sprintf("esxi-%02d.example.org", n), State: report.StateOK, Word: "Connected", Since: now.Add(-time.Duration(30+n) * 24 * time.Hour), Facts: map[string]any{"cpus": 48, "memory_gib": 768.0, "version": "VMware ESXi 8.0.3 build-24414501"}} - switch n { - case 9: - r.State, r.Word, r.Since = report.StateUpdating, "In maintenance mode", now.Add(-35*time.Minute) - case 13: - r.State, r.Word, r.Facts["alarms"] = report.StateDegraded, "vCenter alarm: Host memory usage", "Host memory usage (red)" - } put(r) } for d := 1; d <= 6; d++ { @@ -62,13 +56,18 @@ func newVSphere(name, token string, vms int) *fakeCluster { r.State, r.Word = report.StateStopped, "Powered off" case k == 77: r.State, r.Word, r.Facts["oldest_snapshot"] = report.StateDegraded, "Running, with a snapshot 23 days old", now.Add(-23*24*time.Hour).Format(time.DateOnly) - case k == 150: - r.State, r.Word = report.StateDown, "Inaccessible: its files can't be reached" - r.Since = now.Add(-9 * time.Minute) case k%40 == 5: r.Facts["vmware_tools"] = "not running" } put(r) } + // Patching takes a host into maintenance for a while; a busy one + // raises a memory alarm now and then (scenario.go). + c.flap("host-1009", spellESXiUpdate, func(r *report.Workload, since time.Time) { + r.State, r.Word, r.Since = report.StateUpdating, "In maintenance mode", since + }) + c.flap("host-1013", spellESXiMemory, func(r *report.Workload, since time.Time) { + r.State, r.Word, r.Since, r.Facts["alarms"] = report.StateDegraded, "vCenter alarm: Host memory usage", since, "Host memory usage (red)" + }) return c } diff --git a/scripts/fleet/windows.go b/scripts/fleet/windows.go index 8c4ba79..838e5f2 100644 --- a/scripts/fleet/windows.go +++ b/scripts/fleet/windows.go @@ -14,10 +14,11 @@ import ( // agents named win-*, outside the sites of 50: // // win-dc01 a domain controller, all well -// win-fs01 a file server that rebooted itself 25 minutes ago (Kernel-Power -// 41, critical) and has a disk with bad blocks -// win-sql01 SQL Server: its Agent service should start by itself and has -// stopped; its certificate ends in a few days +// win-fs01 a file server with a disk going bad: bad blocks, and every +// 8 hours it crashes and comes back (Kernel-Power 41, critical) +// win-sql01 SQL Server: its Agent service should start by itself, and +// now and then stops (scenario.go); its certificate ends in a +// few days // win-hv01 a Hyper-V host with twelve VMs: one saved, one failed type winServer struct { name, os, kernel string @@ -73,17 +74,26 @@ func newWindowsHost(name string, interval time.Duration) *host { // them. func (wh *winHost) stamp(h *host, r *report.Report, now time.Time) { w := wh.w + crashes, lastCrash := 0, time.Time{} if w.name == "win-fs01" { - // It went down without shutting down cleanly 25 minutes ago, and - // still has, whenever one looks. - h.start = now.Add(-25 * time.Minute) - r.Uptime = now.Sub(h.start).Seconds() + // It goes down without shutting down cleanly and is straight back: + // up since the last time. + lastCrash = spellFSCrash.began(now) + for t := lastCrash; now.Sub(t) < 24*time.Hour; t = t.Add(-spellFSCrash.every) { + crashes++ + } + h.start = lastCrash + r.Uptime = max(0, now.Sub(h.start).Seconds()) } r.Units = nil for _, u := range w.units { un := report.Unit{Name: u + ".service", Load: "loaded", Active: "active", Sub: "running", Since: h.start.UTC().Add(time.Minute).Truncate(time.Second)} if w.name == "win-sql01" && u == "SQLSERVERAGENT" { - un.Active, un.Sub, un.Since = "failed", "dead", time.Time{} + if b := spellSQLAgent.began(now); spellSQLAgent.active(now) { + un.Active, un.Sub, un.Since = "failed", "dead", time.Time{} + } else if b.After(h.start) { // someone started it again + un.Since = b.Add(spellSQLAgent.on).UTC().Truncate(time.Second) + } } r.Units = append(r.Units, un) } @@ -94,13 +104,14 @@ func (wh *winHost) stamp(h *host, r *report.Report, now time.Time) { } switch w.name { case "win-fs01": - group("System", "Microsoft-Windows-Kernel-Power", 41, "critical", 1, 24*time.Minute, 24*time.Minute, + first := lastCrash.Add(-time.Duration(crashes-1) * spellFSCrash.every) + group("System", "Microsoft-Windows-Kernel-Power", 41, "critical", crashes, now.Sub(first), now.Sub(h.start), "The system has rebooted without cleanly shutting down first. This error could be caused if the system stopped responding, crashed, or lost power unexpectedly.") group("System", "disk", 7, "error", 14, 19*time.Hour, 2*time.Hour, `The device, \Device\Harddisk1\DR1, has a bad block.`) - group("System", "NETLOGON", 5719, "error", 2, 23*time.Minute, 23*time.Minute, + group("System", "NETLOGON", 5719, "error", 2*crashes, now.Sub(first), now.Sub(h.start), "This computer was not able to set up a secure session with a domain controller in domain CORP due to the following: There are currently no logon servers available to service the logon request.") case "win-sql01": - group("System", "Service Control Manager", 7034, "error", 3, 6*time.Hour, 40*time.Minute, + group("System", "Service Control Manager", 7034, "error", 3, 6*time.Hour, now.Sub(spellSQLAgent.began(now)), "The SQL Server Agent (MSSQLSERVER) service terminated unexpectedly. It has done this 3 time(s).") case "win-hv01": group("System", "Microsoft-Windows-Hyper-V-Worker", 18560, "critical", 1, 3*time.Hour, 3*time.Hour, -- 2.54.0