Demo: a plan for what goes wrong, with problems that come and go #48
No files matched your search
@@ -49,3 +49,13 @@ obs-beacond and the simulator, so nothing in it can reach anything else:
|
||||
- The root is read-only and the state is in tmpfs, so restarting the
|
||||
container is a fresh demo; restart it on a timer (weekly is plenty) so
|
||||
history doesn't pile up.
|
||||
|
||||
## What a visitor sees
|
||||
|
||||
Most of the simulated fleet is healthy. What's wrong follows a plan
|
||||
(`scripts/fleet/scenario.go`, and "The demo fleet" in development.md):
|
||||
three stories with a cause and its effects, a few lasting warnings, and
|
||||
problems that come and go through the day, so incidents open, resolve and
|
||||
show up under Recently resolved. Because the plan runs on the clock, two
|
||||
visitors an hour apart see different problems, and a restarted demo shows
|
||||
the same ones the old one would have.
|
||||
+14
-5
@@ -67,11 +67,20 @@ send (ntfy, Discord, PagerDuty, a webhook and email) and lists it on
|
||||
http://127.0.0.1:58450/. `fleet run -h` lists the rest: clusters, VMs,
|
||||
Proxmox, vSphere, Nutanix and Pure arrays.
|
||||
|
||||
Some things are wrong on purpose, so every page has something to show: a
|
||||
container exited, a unit failed, a stale backup, PostgreSQL's leader away
|
||||
from home with a replica behind, an origin failing Cloudflare's health
|
||||
check, a zone on Flexible SSL, and so on. The mail server is healthy, to
|
||||
show what normal looks like.
|
||||
Some things are wrong on purpose, so every page has something to show,
|
||||
and most of the fleet is fine, as a real one is. `scripts/fleet/scenario.go`
|
||||
has the plan: three stories with a cause and its effects (a FlashArray
|
||||
lost a drive; a PostgreSQL replica falls behind and the shop's checkout
|
||||
fails; the legacy datacenter drops out of Akamai GTM), a handful of
|
||||
lasting warnings (a failover nobody moved back, a weak key, an old VM
|
||||
snapshot, a zone on Flexible SSL, and so on), and problems that come and
|
||||
go: a unit that fails and is restarted, a container that exits, a check
|
||||
that flaps, a late backup, a host being patched. Each of those is a
|
||||
*spell*: on for a while once every so often, counted from the Unix epoch,
|
||||
so every part of the simulator agrees on what's happening now and a
|
||||
restarted demo carries on where the clock is. Incidents open and resolve
|
||||
through the day, and a dozen or two are open at any moment. The mail
|
||||
server is healthy, to show what normal looks like.
|
||||
|
||||
## Dependencies and notices
|
||||
|
||||
|
||||
+75
-40
@@ -17,18 +17,19 @@ import (
|
||||
// account on loopback, every signature checked. GTM's datacenters are
|
||||
// the first sites and its servers their load balancers' public
|
||||
// addresses (netmap.go, positions 1 and 2), so a failing one is a fleet
|
||||
// host. Problems on purpose:
|
||||
// host. Problems on purpose, the ones marked * coming and going
|
||||
// (scenario.go):
|
||||
//
|
||||
// GTM site 2's first load balancer failing for api.shop; the
|
||||
// legacy property with no live datacenter; corp's change
|
||||
// propagating for 45 minutes
|
||||
// CPS 180 enrollments: four ending within 21 days (one within
|
||||
// 7), two third-party renewals waiting on a CSR
|
||||
// Property 320 properties: a failed production activation, one
|
||||
// Manager stuck on staging, two rolling out
|
||||
// GTM * site 2's first load balancer failing for api.shop;
|
||||
// * the legacy property with no live datacenter (story 3)
|
||||
// CPS 180 enrollments: a third-party renewal waiting on a CSR,
|
||||
// ending in 15 days
|
||||
// Property 320 properties: a failed production activation;
|
||||
// Manager * one rolling out for longer than it should
|
||||
// Edge DNS 40 zones (one of 5,000 records); one changes an MX 20
|
||||
// minutes after the start
|
||||
// Traffic 320 CP codes; one at 8% 5xx, one at 30%
|
||||
// Traffic 320 CP codes; * the legacy site's at 30% 5xx while its
|
||||
// datacenter is down (story 3)
|
||||
const akamaiPort = 58410
|
||||
|
||||
func serveAkamai(ctx context.Context, dir string) {
|
||||
@@ -48,6 +49,18 @@ func serveAkamai(ctx context.Context, dir string) {
|
||||
f := &akamai.Fake{Creds: c}
|
||||
start := time.Now()
|
||||
n := fleetAkamai(f)
|
||||
go func() {
|
||||
t := time.NewTicker(30 * time.Second)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case now := <-t.C:
|
||||
akamaiStory(f, now)
|
||||
}
|
||||
}
|
||||
}()
|
||||
srv := &http.Server{Handler: f, ReadHeaderTimeout: 5 * time.Second}
|
||||
go func() { _ = srv.Serve(ln) }()
|
||||
go func() { <-ctx.Done(); _ = srv.Close() }()
|
||||
@@ -90,35 +103,25 @@ func fleetAkamai(f *akamai.Fake) string {
|
||||
switch name {
|
||||
case "shop":
|
||||
ps[0].Name = "api"
|
||||
if t := ps[0].Targets[3100+2%sites]; t != nil {
|
||||
t[lb(2%sites, 1)] = false
|
||||
}
|
||||
case "corp":
|
||||
ps = append(ps, akamai.FakeProperty{Name: "legacy", Type: "failover", Targets: map[int]map[string]bool{3100: {"198.51.100.50": false}}})
|
||||
ps = append(ps, akamai.FakeProperty{Name: "legacy", Type: "failover", Targets: map[int]map[string]bool{3100: {legacyServer: true}}})
|
||||
}
|
||||
props += len(ps)
|
||||
fd := akamai.FakeDomain{Type: "full", Datacenters: dcs, Properties: ps}
|
||||
if name == "corp" {
|
||||
fd.Propagation, fd.Since = "PENDING", now.Add(-45*time.Minute)
|
||||
}
|
||||
f.SetDomain(name+".example.akadns.net", fd)
|
||||
f.SetDomain(name+".example.akadns.net", akamai.FakeDomain{Type: "full", Datacenters: dcs, Properties: ps})
|
||||
}
|
||||
// CPS: ECDSA keys (fast to make); every tenth has an RSA one too.
|
||||
for i := range 180 {
|
||||
cn := fmt.Sprintf("app%03d.example.com", i)
|
||||
left := time.Duration(25+(i*37)%300) * day
|
||||
switch i {
|
||||
case 3:
|
||||
left = 5 * day
|
||||
case 11, 47, 101:
|
||||
left = time.Duration(10+i%9) * day
|
||||
if i == 47 {
|
||||
left = 15 * day
|
||||
}
|
||||
e := akamai.FakeEnrollment{CN: cn, SANs: []string{cn, "www." + cn}, Validation: "dv", Contract: "1-FLEET", Waiting: "-"}
|
||||
issuer := "Let's Encrypt E7"
|
||||
if i%6 == 0 {
|
||||
e.Validation, issuer = "ov", "DigiCert Global G2 TLS RSA SHA256 2020 CA1"
|
||||
}
|
||||
if i == 47 || i == 120 {
|
||||
if i == 47 {
|
||||
e.Validation, e.Waiting, issuer = "third-party", "third-party-csr", "Sectigo RSA Domain Validation Secure Server CA"
|
||||
}
|
||||
e.Deployed = []string{akamai.FakePEM(issuer, e.SANs, true, now.Add(left))}
|
||||
@@ -142,15 +145,11 @@ func fleetAkamai(f *akamai.Fake) string {
|
||||
p.Latest, p.Staging = v+1, v+1
|
||||
p.Activations = append(p.Activations, staged,
|
||||
akamai.FakeActivation{Version: v + 1, Network: "PRODUCTION", Status: "FAILED", Submitted: now.Add(-3 * time.Hour), Note: "Origin moved to the new load balancers"})
|
||||
case 9:
|
||||
p.Latest = v + 1
|
||||
p.Activations = append(p.Activations, akamai.FakeActivation{Version: v + 1, Network: "STAGING", Status: "ZONE_1", Submitted: now.Add(-100 * time.Minute)})
|
||||
case 15, 16:
|
||||
p.Latest, p.Staging = v+1, v+1
|
||||
p.Activations = append(p.Activations, staged,
|
||||
akamai.FakeActivation{Version: v + 1, Network: "PRODUCTION", Status: "PENDING", Submitted: now.Add(-time.Duration(i-10) * time.Minute)})
|
||||
}
|
||||
f.SetPMProperty(fmt.Sprintf("prp_9%05d", i), p)
|
||||
if i == slowActivation {
|
||||
slowBase = p
|
||||
}
|
||||
}
|
||||
// Edge DNS.
|
||||
records := 0
|
||||
@@ -164,21 +163,57 @@ func fleetAkamai(f *akamai.Fake) string {
|
||||
records += len(recs)
|
||||
f.SetZone(fmt.Sprintf("zone%02d.example", z), akamai.FakeZone{Records: recs})
|
||||
}
|
||||
// Traffic: a long tail; CP code 400007 at 8% 5xx, 400019 at 30%.
|
||||
// Traffic: a long tail, about one request in 2,000 failing.
|
||||
for i := range 320 {
|
||||
hits := int64(2_000_000 / (i + 1))
|
||||
errs := hits / 2000
|
||||
switch i {
|
||||
case 7:
|
||||
errs = hits * 8 / 100
|
||||
case 19:
|
||||
errs = hits * 30 / 100
|
||||
}
|
||||
f.SetTraffic(int64(400000+i), hits, errs)
|
||||
f.SetTraffic(int64(400000+i), hits, hits/2000)
|
||||
}
|
||||
akamaiStory(f, now)
|
||||
return fmt.Sprintf("%d GTM properties over %d datacenters, 180 CPS enrollments, 320 properties, 40 DNS zones with %d record sets, 320 CP codes", props, sites, records)
|
||||
}
|
||||
|
||||
const (
|
||||
legacyServer = "198.51.100.50" // the legacy property's one datacenter's server
|
||||
legacyCPCode = 400019 // the legacy site's traffic
|
||||
slowActivation = 15 // the property whose production activation drags
|
||||
)
|
||||
|
||||
// slowBase is the slow property as fleetAkamai made it, before its new
|
||||
// version.
|
||||
var slowBase akamai.FakePMProperty
|
||||
|
||||
// akamaiStory puts the problems that come and go in their state now
|
||||
// (scenario.go): story 3, the legacy datacenter; a load balancer that
|
||||
// fails GTM's liveness test now and then; an activation that drags.
|
||||
func akamaiStory(f *akamai.Fake, now time.Time) {
|
||||
legacyDown := spellLegacyDC.active(now)
|
||||
f.SetAlive("corp.example.akadns.net", "legacy", 3100, legacyServer, !legacyDown)
|
||||
hits := int64(2_000_000 / (legacyCPCode - 400000 + 1))
|
||||
errs := hits / 2000
|
||||
if legacyDown {
|
||||
errs = hits * 30 / 100
|
||||
}
|
||||
f.SetTraffic(legacyCPCode, hits, errs)
|
||||
if sites := min(fleetSites, 6); sites > 2 {
|
||||
f.SetAlive("shop.example.akadns.net", "api", 3100+2, fmt.Sprintf("203.0.%d.%d", 2, 11), !spellLBFlaps.active(now))
|
||||
}
|
||||
// Its new version goes to production at the start of each spell and
|
||||
// is active an hour and twenty minutes later.
|
||||
p := slowBase
|
||||
p.Activations = append([]akamai.FakeActivation{}, p.Activations...)
|
||||
v := p.Latest + 1
|
||||
b := spellActivation.began(now)
|
||||
p.Latest, p.Staging = v, v
|
||||
p.Activations = append(p.Activations, akamai.FakeActivation{Version: v, Network: "STAGING", Status: "ACTIVE", Submitted: b.Add(-2 * time.Hour)})
|
||||
if spellActivation.active(now) {
|
||||
p.Activations = append(p.Activations, akamai.FakeActivation{Version: v, Network: "PRODUCTION", Status: "PENDING", Submitted: b})
|
||||
} else {
|
||||
p.Production = v
|
||||
p.Activations = append(p.Activations, akamai.FakeActivation{Version: v, Network: "PRODUCTION", Status: "ACTIVE", Submitted: b})
|
||||
}
|
||||
f.SetPMProperty(fmt.Sprintf("prp_9%05d", slowActivation), p)
|
||||
}
|
||||
|
||||
// fleetZone is zone z's ordinary records, with the given MX.
|
||||
func fleetZone(z int, mx string) []akamai.RecordSet {
|
||||
name := fmt.Sprintf("zone%02d.example", z)
|
||||
|
||||
@@ -19,8 +19,7 @@ import (
|
||||
// last one though its file was renewed, and site 5's sends it
|
||||
// without its intermediate
|
||||
// 30-39 PostgreSQL: each its own certificate from the internal CA
|
||||
// (STARTTLS), and in /etc/pki/tls/certs; a few end soon, one
|
||||
// has a 1024-bit key
|
||||
// (STARTTLS), and in /etc/pki/tls/certs; one has a 1024-bit key
|
||||
//
|
||||
// Clusters have the router's (or ingress controller's) wildcard on every
|
||||
// Route or Ingress, and cert-manager Certificates for ten of their apps.
|
||||
@@ -76,12 +75,6 @@ func hostCerts(i int, name string) (*report.CertScan, *report.CertFiles) {
|
||||
file("/etc/haproxy/certs/example.com.pem", c, wildcard.NotBefore.Add(time.Hour))
|
||||
case pos >= 30 && pos < 40:
|
||||
begun := time.Duration(30+(n*37)%300) * 24 * time.Hour
|
||||
switch {
|
||||
case n%97 == 30: // ends in a few days
|
||||
begun = 360 * 24 * time.Hour
|
||||
case n%131 == 31: // ended
|
||||
begun = 370 * 24 * time.Hour
|
||||
}
|
||||
key := "ECDSA P-256"
|
||||
if n%211 == 32 {
|
||||
key = "RSA 1024"
|
||||
@@ -158,7 +151,7 @@ func clusterCerts(name string, ocp bool, routes int, put func(report.Workload))
|
||||
for n := range min(10, routes) {
|
||||
app := appNames[n]
|
||||
host := app + ".example.com"
|
||||
begun := time.Duration(10+(n*7+k*3)%60) * 24 * time.Hour
|
||||
begun := time.Duration(10+(n*7+k*3)%55) * 24 * time.Hour // all a month or more from their end, renewing at 30 days
|
||||
if n == 4 && k == 0 {
|
||||
begun = 86 * 24 * time.Hour // ends in four days: the renewal is failing
|
||||
}
|
||||
|
||||
@@ -17,11 +17,10 @@ import (
|
||||
// dozen zones, a load-balanced pool per site whose origins are the sites'
|
||||
// load balancers, and the last 15 minutes of traffic. obs-beacond reads it
|
||||
// through cloudflare's stand_in (http on loopback only). Problems on
|
||||
// purpose: legacy.example still uses Flexible SSL, partners.example's edge
|
||||
// certificate ends in 9 days, media.example's advanced certificate is
|
||||
// stuck validating, fleet-0052 fails its pool's health check from eastern
|
||||
// North America, and 7% of shop.example's requests fail, most on
|
||||
// checkout.shop.example.
|
||||
// purpose: legacy.example still uses Flexible SSL; now and then fleet-0052
|
||||
// fails its pool's health check from eastern North America; and while
|
||||
// the shop's database replica is behind (scenario.go, story 2) 7% of
|
||||
// shop.example's requests fail, most on checkout.shop.example.
|
||||
const cloudflarePort = 58430
|
||||
|
||||
// cfToken is the fake's API token; not a secret, as the fake is loopback only.
|
||||
@@ -109,16 +108,8 @@ func cfHandle(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
ends := now.Add(time.Duration(40+k*3) * 24 * time.Hour)
|
||||
if z == "partners.example" {
|
||||
ends = now.Add(9*24*time.Hour + 5*time.Hour)
|
||||
}
|
||||
packs := []map[string]any{{"type": "universal", "status": "active", "hosts": []string{z, "*." + z},
|
||||
"certificates": []map[string]string{{"expires_on": ends.Format(time.RFC3339)}}}}
|
||||
if z == "media.example" {
|
||||
packs = append(packs, map[string]any{"type": "advanced", "status": "pending_validation", "hosts": []string{"cdn." + z},
|
||||
"certificates": []map[string]string{}})
|
||||
}
|
||||
ok(packs)
|
||||
ok([]map[string]any{{"type": "universal", "status": "active", "hosts": []string{z, "*." + z},
|
||||
"certificates": []map[string]string{{"expires_on": ends.Format(time.RFC3339)}}}})
|
||||
case len(parts) == 3 && parts[0] == "zones" && parts[2] == "load_balancers":
|
||||
_, z, found := zone(parts[1])
|
||||
if !found {
|
||||
@@ -169,7 +160,7 @@ func cfHandle(w http.ResponseWriter, r *http.Request) {
|
||||
for pos := 1; pos <= 2; pos++ {
|
||||
_, a := cfOrigin(s, pos)
|
||||
h := map[string]any{"healthy": true, "rtt": fmt.Sprintf("%dms", 20+ri*30+pos*3), "failure_reason": "No failures", "response_code": 200}
|
||||
if s == 1 && pos == 2 && region == "ENAM" {
|
||||
if s == 1 && pos == 2 && region == "ENAM" && spellLBFlaps.active(now) {
|
||||
h = map[string]any{"healthy": false, "rtt": "0ms", "failure_reason": "Response code mismatch error", "response_code": 503}
|
||||
}
|
||||
origins = append(origins, map[string]any{a: h})
|
||||
@@ -197,7 +188,7 @@ func cfHandle(w http.ResponseWriter, r *http.Request) {
|
||||
return map[string]any{"count": n, "dimensions": map[string]any{"edgeResponseStatus": status, "securityAction": action, "clientRequestHTTPHost": host}}
|
||||
}
|
||||
win := []map[string]any{g(base, 200, "unknown", z), g(base/20, 304, "unknown", "www."+z), g(3, 502, "unknown", z), g(base/400, 403, "block", z)}
|
||||
if z == "shop.example" {
|
||||
if z == "shop.example" && spellReplicaBehind.active(now) {
|
||||
win = append(win, g(base*6/100, 503, "unknown", "checkout.shop.example"), g(base/100, 502, "unknown", z))
|
||||
}
|
||||
zs = append(zs, map[string]any{"zoneTag": id, "win": win, "day": []map[string]int{{"count": base / 400 * 96}}})
|
||||
|
||||
+111
-6
@@ -16,6 +16,7 @@ import (
|
||||
"math"
|
||||
mrand "math/rand/v2"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -47,6 +48,105 @@ type fakeCluster struct {
|
||||
noUsage bool // its reader sends none (Nutanix: usage is a separate API, not read yet)
|
||||
source string // what a host agent calls it (a vCenter's host name); "" for a cluster agent
|
||||
usageFn func(now time.Time) []report.Usage // its own readings instead of CPU and memory (an array's performance)
|
||||
episodes []episode // records that go wrong and come right (scenario.go)
|
||||
}
|
||||
|
||||
// episode is a record that takes its bad form while its spell is on.
|
||||
type episode struct {
|
||||
id string
|
||||
s spell
|
||||
good report.Workload
|
||||
bad func(w *report.Workload, since time.Time)
|
||||
on, in bool // on: in its bad form; in: played at least once
|
||||
}
|
||||
|
||||
// flap makes record id go wrong during s and come right after
|
||||
// (scenario.go): bad changes a copy of the record as it is now, given
|
||||
// when the spell began.
|
||||
func (c *fakeCluster) flap(id string, s spell, bad func(w *report.Workload, since time.Time)) {
|
||||
if _, ok := c.recs[id]; !ok {
|
||||
panic("flap: no record " + id)
|
||||
}
|
||||
c.episodes = append(c.episodes, episode{id: id, s: s, good: cloneWorkload(c.recs[id]), bad: bad})
|
||||
c.play(time.Now())
|
||||
}
|
||||
|
||||
// flapSet flaps every record that bad has and good hasn't, or has
|
||||
// otherwise: a reader's records made from a good and a bad snapshot. A
|
||||
// record only bad has is there during the spell and gone after.
|
||||
func (c *fakeCluster) flapSet(s spell, good, bad []report.Workload) {
|
||||
have := map[string]report.Workload{}
|
||||
for _, r := range good {
|
||||
have[r.ID] = r
|
||||
}
|
||||
for _, b := range bad {
|
||||
g, ok := have[b.ID]
|
||||
if ok {
|
||||
g.Since, b.Since = time.Time{}, time.Time{}
|
||||
if reflect.DeepEqual(g, b) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
b := cloneWorkload(b)
|
||||
if !ok {
|
||||
c.episodes = append(c.episodes, episode{id: b.ID, s: s, bad: func(w *report.Workload, since time.Time) { *w = cloneWorkload(b); w.Since = since }})
|
||||
continue
|
||||
}
|
||||
c.flap(b.ID, s, func(w *report.Workload, since time.Time) { *w = cloneWorkload(b); w.Since = since })
|
||||
}
|
||||
c.play(time.Now())
|
||||
}
|
||||
|
||||
// play puts each flapping record in the form its spell says, and marks
|
||||
// the ones that changed.
|
||||
func (c *fakeCluster) play(now time.Time) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
for i := range c.episodes {
|
||||
e := &c.episodes[i]
|
||||
on := e.s.active(now)
|
||||
if e.in && on == e.on {
|
||||
continue
|
||||
}
|
||||
e.on, e.in = on, true
|
||||
if e.good.ID == "" && !on { // there only during the spell
|
||||
if _, ok := c.recs[e.id]; ok {
|
||||
delete(c.recs, e.id)
|
||||
delete(c.changed, e.id)
|
||||
c.removed = append(c.removed, e.id)
|
||||
}
|
||||
continue
|
||||
}
|
||||
w := cloneWorkload(e.good)
|
||||
if on {
|
||||
e.bad(&w, e.s.began(now))
|
||||
} else if !e.good.Since.IsZero() && now.Sub(e.s.began(now)) < e.s.every {
|
||||
w.Since = e.s.began(now).Add(e.s.on) // right again since the spell ended
|
||||
}
|
||||
c.recs[e.id] = w
|
||||
c.changed[e.id] = true
|
||||
}
|
||||
}
|
||||
|
||||
// flapping says whether record id has an episode, so churn leaves it be.
|
||||
func (c *fakeCluster) flapping(id string) bool {
|
||||
for _, e := range c.episodes {
|
||||
if e.id == id {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func cloneWorkload(w report.Workload) report.Workload {
|
||||
if w.Facts != nil {
|
||||
f := make(map[string]any, len(w.Facts))
|
||||
for k, v := range w.Facts {
|
||||
f[k] = v
|
||||
}
|
||||
w.Facts = f
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
func newID() string {
|
||||
@@ -69,6 +169,7 @@ func newCluster(name, token string, s clusterShape) *fakeCluster {
|
||||
Since: now.Add(-48 * time.Hour), Facts: map[string]any{"kubelet": kubelet, "os": nodeOS, "cpus": 32, "memory_gib": 256}})
|
||||
}
|
||||
k := clusterIndex(name)
|
||||
crasher := ""
|
||||
for i := range s.controllers {
|
||||
ns, owner := fmt.Sprintf("ns-%02d", i%30), fmt.Sprintf("deployment/app-%03d", i)
|
||||
want := s.pods / max(s.controllers, 1)
|
||||
@@ -82,11 +183,8 @@ func newCluster(name, token string, s clusterShape) *fakeCluster {
|
||||
for range want {
|
||||
c.addPod(ns, owner, now)
|
||||
}
|
||||
if i%97 == 0 { // a few controllers with a crashing pod
|
||||
id := c.pods[len(c.pods)-1]
|
||||
p := c.recs[id]
|
||||
p.State, p.Word, p.Facts = report.StateCrashloop, "Restarting (CrashLoopBackOff)", map[string]any{"restarts": 40, "last_exit": "Error"}
|
||||
put(p)
|
||||
if i == 97 && k == 0 { // a pod that crashes for a while after each deploy (scenario.go)
|
||||
crasher = c.pods[len(c.pods)-1]
|
||||
}
|
||||
}
|
||||
vmGroup := map[string][]string{}
|
||||
@@ -142,6 +240,12 @@ func newCluster(name, token string, s clusterShape) *fakeCluster {
|
||||
if name == "cluster-00" { // the pod the probe changes
|
||||
put(report.Workload{Kind: report.KindPod, ID: "probe", Name: "probe", Namespace: "fleet", Node: "node-00", State: report.StateOK, Word: "Running"})
|
||||
}
|
||||
if crasher != "" {
|
||||
c.flap(crasher, spellCrashloop, func(p *report.Workload, since time.Time) {
|
||||
p.State, p.Word, p.Since = report.StateCrashloop, "Restarting (CrashLoopBackOff)", since
|
||||
p.Facts["restarts"], p.Facts["last_exit"], p.Facts["ready"] = 9, "Error", "0/1"
|
||||
})
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -212,7 +316,7 @@ func (c *fakeCluster) churn(n int, now time.Time) {
|
||||
i := mrand.N(len(c.pods))
|
||||
id := c.pods[i]
|
||||
p := c.recs[id]
|
||||
if p.State == report.StateCrashloop {
|
||||
if p.State == report.StateCrashloop || c.flapping(id) {
|
||||
continue
|
||||
}
|
||||
if mrand.N(10) == 0 { // replaced: a new pod, the old one gone
|
||||
@@ -372,6 +476,7 @@ func runCluster(ctx context.Context, c *fakeCluster, client *http.Client, pick f
|
||||
}
|
||||
}
|
||||
c.churn(int(c.shape.churn*5), now)
|
||||
c.play(now)
|
||||
if probed != nil {
|
||||
probeWord, probeAt = c.probe(now), now
|
||||
}
|
||||
|
||||
@@ -20,7 +20,9 @@ import (
|
||||
// real network; the status pages come in the hosts' reports, as an agent's
|
||||
// fetches do.
|
||||
// Problems on purpose: the leader is fleet-0031, not its usual fleet-0030
|
||||
// (a failover nobody moved back), and fleet-0032 is 40 MB behind.
|
||||
// (a failover nobody moved back), and every so often fleet-0032 falls 40
|
||||
// MB behind and HAProxy takes it out of the replicas' pool until it
|
||||
// catches up (scenario.go, story 2).
|
||||
const patroniPort = 58420
|
||||
|
||||
var (
|
||||
@@ -85,7 +87,9 @@ func servePatroni(ctx context.Context) {
|
||||
case pgHome:
|
||||
role = "sync_standby"
|
||||
case pgLagging:
|
||||
lag = `,"lag":41943040`
|
||||
if spellReplicaBehind.active(time.Now()) {
|
||||
lag = `,"lag":41943040`
|
||||
}
|
||||
}
|
||||
ms = append(ms, fmt.Sprintf(`{"name":%q,"role":%q,"state":%q,"host":%q,"port":5432,"timeline":7%s}`, hostName(m), role, state, pgAddr(m), lag))
|
||||
}
|
||||
@@ -131,7 +135,8 @@ func dataFetches(n int, now time.Time) []report.Fetch {
|
||||
fmt.Fprintf(&b, "%s,BACKEND,0,UP,1\n", name)
|
||||
}
|
||||
backend("pg-primary", func(m int) bool { return m == pgLeader }, pgMembers)
|
||||
backend("pg-replicas", func(m int) bool { return m != pgLeader }, pgMembers)
|
||||
behind := spellReplicaBehind.active(now)
|
||||
backend("pg-replicas", func(m int) bool { return m != pgLeader && !(behind && m == pgLagging) }, pgMembers)
|
||||
var apps []int
|
||||
for p := 3; p < 13; p++ {
|
||||
apps = append(apps, p)
|
||||
|
||||
+72
-21
@@ -16,17 +16,20 @@ import (
|
||||
// keep running, and the backups each site keeps, so that every page has
|
||||
// something to show. config writes the files obs-beacond reads; run makes
|
||||
// the hosts report against them. Both work them out from the number of
|
||||
// agents, so give both the same -n. Problems on purpose:
|
||||
// agents, so give both the same -n. Problems on purpose, most of them
|
||||
// coming and going (scenario.go):
|
||||
//
|
||||
// checks legacy.example.com answers 502 (a warning-only check);
|
||||
// billing.example.com's certificate ends in 9 days; the
|
||||
// ping to the last site loses 3%
|
||||
// units fleet-0041's redis-server has failed
|
||||
// containers fleet-0007's app-04 exited; fleet-0008's app-09 is stopped
|
||||
// on purpose
|
||||
// backups site 1's newest pg-dump is 40 hours old; site 2's shrank to
|
||||
// a third; site 3's backup disk is 93% full; site 4's
|
||||
// pg-backup job failed; site 5's md1 is missing a disk
|
||||
// checks legacy.example.com answers 502 now and then (a warning-only
|
||||
// check); billing.example.com's certificate ends in 9 days;
|
||||
// the ping to the last site loses 3%; mx2 lands on a block
|
||||
// list now and then
|
||||
// units fleet-0041's redis-server fails now and then, and is
|
||||
// restarted
|
||||
// containers fleet-0007's app-04 exits now and then; fleet-0008's app-09
|
||||
// is stopped on purpose
|
||||
// backups site 1's pg-dump is late now and then; site 2's shrinks to a
|
||||
// third now and then; site 3's backup disk is 93% full; site
|
||||
// 4's pg-backup job failed; site 5's md1 is missing a disk
|
||||
|
||||
// role is what a host does in its site, by position (netmap.go).
|
||||
func role(pos int) string {
|
||||
@@ -112,6 +115,9 @@ func demoChecks(hosts int) []demoCheck {
|
||||
out = append(out,
|
||||
demoCheck{id: "web-legacy", name: "legacy.example.com", yaml: "http: https://legacy.example.com/\n severity: warning", from: web,
|
||||
result: func(from int, now time.Time) report.CheckResult {
|
||||
if !spellLegacy502.active(now) {
|
||||
return report.CheckResult{ID: "web-legacy", At: now.UTC(), OK: true, Status: 200, Ms: 340}
|
||||
}
|
||||
return report.CheckResult{ID: "web-legacy", At: now.UTC(), Status: 502, Ms: 2100, Error: "HTTP 502, want 200"}
|
||||
}},
|
||||
demoCheck{id: "dns-mx", name: "example.com MX", yaml: "dns: example.com\n type: MX\n contains: mx1.example.com", from: web,
|
||||
@@ -126,6 +132,9 @@ func demoChecks(hosts int) []demoCheck {
|
||||
// mx2 landed on a list that flags whole networks: a warning to look at, not a page.
|
||||
demoCheck{id: "rbl-mx2", name: "mx2.example.com on block lists", yaml: "rbl: 192.0.2.26\n severity: warning", from: web[:1],
|
||||
result: func(from int, now time.Time) report.CheckResult {
|
||||
if !spellBlocklisted.active(now) {
|
||||
return report.CheckResult{ID: "rbl-mx2", At: now.UTC(), OK: true, Ms: 190, Answer: "clean on 5 of 5"}
|
||||
}
|
||||
return report.CheckResult{ID: "rbl-mx2", At: now.UTC(), Ms: 210, Answer: "clean on 4 of 5",
|
||||
Error: "listed on dnsbl-1.uceprotect.net (127.0.0.2)"}
|
||||
}},
|
||||
@@ -204,16 +213,19 @@ func demoScan(n, hosts int, now time.Time) *report.Backups {
|
||||
if name == "system" {
|
||||
size = int64(800+40*k) << 20
|
||||
}
|
||||
d := report.BackupDir{Name: name, From: hostName(from), Path: fmt.Sprintf("/srv/backups/%s/%s/%s", hostName(from), name, now.Format("2006-01-02")),
|
||||
Size: size, PrevSize: size - size/50, Count: 14, Modified: now.Add(-time.Duration(3+k) * time.Hour).UTC()}
|
||||
d := report.BackupDir{Name: name, From: hostName(from), Size: size, PrevSize: size - size/50, Count: 14,
|
||||
Modified: now.Add(-time.Duration(3+k) * time.Hour).UTC()}
|
||||
if k == 0 && name == "pg-dump" {
|
||||
switch site {
|
||||
case 1:
|
||||
d.Modified = now.Add(-40 * time.Hour).UTC()
|
||||
case 2:
|
||||
switch {
|
||||
case site == 1 && spellLateBackup.active(now):
|
||||
// Last night's never came: the newest is the night before's.
|
||||
d.Modified = spellLateBackup.began(now).Add(-26*time.Hour - 10*time.Minute).UTC()
|
||||
case site == 2 && spellBackupShrank.active(now):
|
||||
d.PrevSize, d.Size = d.Size, d.Size/3
|
||||
}
|
||||
}
|
||||
// Each copy is in a folder named after the day it was made.
|
||||
d.Path = fmt.Sprintf("/srv/backups/%s/%s/%s", hostName(from), name, d.Modified.Format("2006-01-02"))
|
||||
sc.Received = append(sc.Received, d)
|
||||
}
|
||||
}
|
||||
@@ -328,22 +340,60 @@ func (d *demoHost) setup(r *report.Report) {
|
||||
}
|
||||
for k := range r.Containers.List {
|
||||
c := &r.Containers.List[k]
|
||||
switch {
|
||||
case d.n == demoExited && c.Name == "app-04":
|
||||
c.State, c.Status, c.Health = "exited", "Exited (137) 12 minutes ago", ""
|
||||
case d.n == demoStoppedOnPurp && c.Name == "app-09":
|
||||
if d.n == demoStoppedOnPurp && c.Name == "app-09" {
|
||||
c.State, c.Status, c.Health = "exited", "Exited (0) 2 days ago", ""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// containers puts the containers that come and go in their state now.
|
||||
func (d *demoHost) containers(r *report.Report, now time.Time) {
|
||||
if d.n != demoExited || r.Containers == nil {
|
||||
return
|
||||
}
|
||||
list := append([]report.Container{}, r.Containers.List...)
|
||||
for k := range list {
|
||||
c := &list[k]
|
||||
if c.Name != "app-04" {
|
||||
continue
|
||||
}
|
||||
if b := spellAppExits.began(now); spellAppExits.active(now) {
|
||||
c.State, c.Status, c.Health = "exited", fmt.Sprintf("Exited (137) %s ago", fuzzy(now.Sub(b))), ""
|
||||
} else {
|
||||
c.State, c.Health = "running", "healthy"
|
||||
c.Status = fmt.Sprintf("Up %s (healthy)", fuzzy(now.Sub(b.Add(spellAppExits.on))))
|
||||
}
|
||||
}
|
||||
r.Containers = &report.Containers{List: list}
|
||||
}
|
||||
|
||||
// fuzzy is a duration as docker ps says it.
|
||||
func fuzzy(d time.Duration) string {
|
||||
switch {
|
||||
case d < time.Minute:
|
||||
return "Less than a minute"
|
||||
case d < 2*time.Minute:
|
||||
return "About a minute"
|
||||
case d < time.Hour:
|
||||
return fmt.Sprintf("%d minutes", int(d.Minutes()))
|
||||
case d < 2*time.Hour:
|
||||
return "About an hour"
|
||||
}
|
||||
return fmt.Sprintf("%d hours", int(d.Hours()))
|
||||
}
|
||||
|
||||
// stamp adds the units, check results and backup scan to a report.
|
||||
func (d *demoHost) stamp(r *report.Report, start, now time.Time) {
|
||||
r.Units = nil
|
||||
for _, u := range demoUnits(d.n) {
|
||||
un := report.Unit{Name: u + ".service", Load: "loaded", Active: "active", Sub: "running", Since: start.UTC()}
|
||||
if d.n == demoFailedUnit && u == "redis-server" {
|
||||
un.Active, un.Sub, un.Restarts, un.Since = "failed", "failed", 5, time.Time{}
|
||||
switch b := spellRedisFails.began(now); {
|
||||
case spellRedisFails.active(now):
|
||||
un.Active, un.Sub, un.Restarts, un.Since = "failed", "failed", 5, time.Time{}
|
||||
case b.After(start): // restarted by hand when the spell ended
|
||||
un.Since = b.Add(spellRedisFails.on).UTC()
|
||||
}
|
||||
}
|
||||
r.Units = append(r.Units, un)
|
||||
}
|
||||
@@ -351,6 +401,7 @@ func (d *demoHost) stamp(r *report.Report, start, now time.Time) {
|
||||
for _, c := range d.checks {
|
||||
r.Checks = append(r.Checks, c.result(d.n, now))
|
||||
}
|
||||
d.containers(r, now)
|
||||
r.Backups = demoScan(d.n, d.hosts, now)
|
||||
if hasDataLayer(d.hosts) {
|
||||
r.Fetches = dataFetches(d.n, now)
|
||||
|
||||
@@ -2,7 +2,6 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/cairnobs/cairnobs/internal/hmc"
|
||||
@@ -18,9 +17,9 @@ const hmcName = "hmc1"
|
||||
|
||||
// newHMC is three servers: p10-prod runs the four IBM i partitions (their
|
||||
// serial and partition numbers are the ones the partitions say), an AIX
|
||||
// database and a Linux pair; p10-dr has its attention light on, a VIOS
|
||||
// the HMC can't reach over RMC and a Linux partition stopped in Open
|
||||
// Firmware; p9-old is in standby, waiting to be retired.
|
||||
// database and a Linux pair; p10-dr has its attention light on (a
|
||||
// serviceable event nobody has closed); p9-old is in standby, waiting to
|
||||
// be retired.
|
||||
func newHMC(name, token string) *fakeCluster {
|
||||
c := &fakeCluster{name: name, token: token, recs: map[string]report.Workload{}, changed: map[string]bool{}, noUsage: true, source: hmcName}
|
||||
n := 0
|
||||
@@ -52,9 +51,8 @@ func newHMC(name, token string) *fakeCluster {
|
||||
MachineType: "9105", Model: "22B", Serial: "78DEF45", ProcUnits: 20, ProcFree: 12.25, MemMB: 1 << 20, MemFreeMB: 655_360, AttentionLED: "true",
|
||||
Partitions: []hmc.Partition{vios("p10-dr", 1, 1), vios("p10-dr", 2, 2),
|
||||
part("p10-dr", "aixdb1-dr", "AIX/Linux", "AIX 7.3 7300-03-00-2446", "running", 3, 2, 8, 512),
|
||||
part("p10-dr", "lnxtest1", "AIX/Linux", "Linux/SUSE 6.4.0-150600.23.25-default 15.6", "open firmware", 4, 0.5, 2, 16),
|
||||
part("p10-dr", "lnxtest1", "AIX/Linux", "Linux/SUSE 6.4.0-150600.23.25-default 15.6", "running", 4, 0.5, 2, 16),
|
||||
part("p10-dr", "aixtest1", "AIX/Linux", "Unknown", "not activated", 5, 0.5, 1, 16)}}
|
||||
dr.Partitions[1].RMC = "inactive"
|
||||
old := hmc.Server{UUID: "1f7a9c3e-5b2d-3e84-a6c0-94d2e8b1f733", Name: "p9-old", State: "standby", DetailedState: "None",
|
||||
MachineType: "9009", Model: "42A", Serial: "13XYZ99", ProcUnits: 16, ProcFree: 16, MemMB: 512 << 10, MemFreeMB: 512 << 10, AttentionLED: "false",
|
||||
Partitions: []hmc.Partition{part("p9-old", "aixold1", "AIX/Linux", "AIX 7.2 7200-05-08-2420", "not activated", 1, 2, 4, 64)}}
|
||||
@@ -62,9 +60,6 @@ func newHMC(name, token string) *fakeCluster {
|
||||
now := time.Now()
|
||||
for i, r := range hmc.Records(s) {
|
||||
r.Since = now.Add(-time.Duration(30+i%11) * 24 * time.Hour)
|
||||
if strings.HasPrefix(r.Word, "Stopped in Open Firmware") || strings.HasPrefix(r.Word, "Running, but the HMC") {
|
||||
r.Since = now.Add(-time.Duration(2+i%5) * time.Hour)
|
||||
}
|
||||
c.recs[r.ID] = r
|
||||
}
|
||||
return c
|
||||
|
||||
+57
-45
@@ -20,12 +20,11 @@ const ibmiHost = 41
|
||||
// ibmiPartitions are the partitions, in the order -ibmi N takes them.
|
||||
var ibmiPartitions = []string{"prod1", "prod2", "ha1", "dev1"}
|
||||
|
||||
// newIBMi is the k'th partition: prod1 busy, its system ASP past IBM i's
|
||||
// threshold, a disk unit's parity being rebuilt and the nightly batch
|
||||
// stuck on a question; prod2 with ORDERS ended, its job queue filling,
|
||||
// and its backup interface down; ha1 the PowerHA standby, its
|
||||
// independent ASP varied off; dev1 on 7.4 without the authority to read
|
||||
// its Technology Refresh.
|
||||
// newIBMi is the k'th partition: prod1 busy, its nightly batch stuck on a
|
||||
// question now and then; prod2's ORDERS subsystem ended for a while now
|
||||
// and then, its job queue filling meanwhile (scenario.go); ha1 the
|
||||
// PowerHA standby, its independent ASP varied off; dev1 on 7.4 without
|
||||
// the authority to read its Technology Refresh.
|
||||
func newIBMi(name, token string, k int) *fakeCluster {
|
||||
part := ibmiPartitions[k]
|
||||
c := &fakeCluster{name: name, token: token, recs: map[string]report.Workload{}, changed: map[string]bool{}, noUsage: true, source: part}
|
||||
@@ -39,49 +38,62 @@ func newIBMi(name, token string, k int) *fakeCluster {
|
||||
pct := min(100, base*(0.6+0.4*day)*(0.9+0.2*rand.Float64()))
|
||||
return []report.Usage{{ID: report.ClusterSelf, CPU: math.Round(pct*units) / 100, CPUOf: units}}
|
||||
}
|
||||
sysUsed := []float64{91.4, 72.8, 64.1, 48.0}[k]
|
||||
s := &ibmi.Snapshot{Name: part, Release: "7.5", TR: 6, At: time.Now(),
|
||||
System: ibmi.System{HostName: "S" + part, PartitionName: strings.ToUpper(part), PartitionID: k + 2, Partitions: 6, MachineType: "9105", MachineModel: "42A",
|
||||
Serial: "78ABC12", CPUs: int(units * 2), Capacity: units, MainStorageKB: int64(units*64) << 20, SystemASPMB: 2_400_000,
|
||||
SystemASPUsed: sysUsed, TempMB: 38_000, Jobs: []int{18204, 14550, 6120, 3307}[k], MaxJobs: 163520,
|
||||
ActiveJobs: []int{612, 488, 201, 97}[k], Threads: []int{2870, 2210, 890, 402}[k]},
|
||||
ASPs: []ibmi.ASP{{Number: 1, State: "NONE", Type: "SYSTEM", Units: 8, Present: "ALL", TotalMB: 2_400_000,
|
||||
FreeMB: int64(2_400_000 * (100 - sysUsed) / 100), Threshold: 90}}}
|
||||
switch part {
|
||||
case "prod1", "ha1":
|
||||
ia := ibmi.ASP{Number: 33, Name: "PRODIASP", State: "AVAILABLE", Type: "PRIMARY", Units: 6, Present: "ALL", TotalMB: 3_600_000, FreeMB: 1_440_000, Threshold: 90}
|
||||
if part == "ha1" {
|
||||
ia.State = "VARIED OFF"
|
||||
sysUsed := []float64{84.6, 72.8, 64.1, 48.0}[k]
|
||||
snap := func(bad bool) *ibmi.Snapshot {
|
||||
s := &ibmi.Snapshot{Name: part, Release: "7.5", TR: 6, At: time.Now(),
|
||||
System: ibmi.System{HostName: "S" + part, PartitionName: strings.ToUpper(part), PartitionID: k + 2, Partitions: 6, MachineType: "9105", MachineModel: "42A",
|
||||
Serial: "78ABC12", CPUs: int(units * 2), Capacity: units, MainStorageKB: int64(units*64) << 20, SystemASPMB: 2_400_000,
|
||||
SystemASPUsed: sysUsed, TempMB: 38_000, Jobs: []int{18204, 14550, 6120, 3307}[k], MaxJobs: 163520,
|
||||
ActiveJobs: []int{612, 488, 201, 97}[k], Threads: []int{2870, 2210, 890, 402}[k]},
|
||||
ASPs: []ibmi.ASP{{Number: 1, State: "NONE", Type: "SYSTEM", Units: 8, Present: "ALL", TotalMB: 2_400_000,
|
||||
FreeMB: int64(2_400_000 * (100 - sysUsed) / 100), Threshold: 90}}}
|
||||
switch part {
|
||||
case "prod1", "ha1":
|
||||
ia := ibmi.ASP{Number: 33, Name: "PRODIASP", State: "AVAILABLE", Type: "PRIMARY", Units: 6, Present: "ALL", TotalMB: 3_600_000, FreeMB: 1_440_000, Threshold: 90}
|
||||
if part == "ha1" {
|
||||
ia.State = "VARIED OFF"
|
||||
}
|
||||
s.ASPs = append(s.ASPs, ia)
|
||||
case "dev1":
|
||||
s.Release, s.TR = "7.4", 0
|
||||
}
|
||||
s.ASPs = append(s.ASPs, ia)
|
||||
case "dev1":
|
||||
s.Release, s.TR = "7.4", 0
|
||||
}
|
||||
sbs := func(lib, name, status string, jobs int) ibmi.Subsystem {
|
||||
return ibmi.Subsystem{Library: lib, Name: name, Status: status, Jobs: jobs, Controlling: name == "QCTL"}
|
||||
}
|
||||
s.Subsys = []ibmi.Subsystem{sbs("QSYS", "QCTL", "ACTIVE", 3), sbs("QSYS", "QINTER", "ACTIVE", 20+k*7), sbs("QSYS", "QBATCH", "ACTIVE", 6+k),
|
||||
sbs("QSYS", "QUSRWRK", "ACTIVE", 60+k*9), sbs("QSYS", "QSERVER", "ACTIVE", 14), sbs("QSYS", "QSYSWRK", "ACTIVE", 110), sbs("QSYS", "QSPL", "ACTIVE", 4),
|
||||
sbs("QSYS", "QHTTPSVR", "ACTIVE", 9), sbs("QSYS", "QSNADS", "INACTIVE", 0), sbs("QSYS", "QCMN", "ACTIVE", 2), sbs("ORDLIB", "ORDERS", "ACTIVE", 12)}
|
||||
s.Ifaces = []ibmi.Interface{{Address: fmt.Sprintf("10.0.4.%d", 20+k), Prefix: 24, Line: "ETHLINE", Status: "ACTIVE", Autostart: true},
|
||||
{Address: fmt.Sprintf("10.0.9.%d", 20+k), Prefix: 24, Line: "ETHBKP", Status: "ACTIVE", Autostart: true}}
|
||||
switch part {
|
||||
case "prod1":
|
||||
s.Disks = []ibmi.Disk{{ASP: 1, Unit: 6, Resource: "DMP006", Hardware: "PARITY REBUILD"}}
|
||||
s.Inquiries = []ibmi.Message{{Key: "0001A2F0", ID: "CPA0701", Inquiry: true, Severity: 99, Job: "418276/BATCH/NIGHTLY", User: "BATCH",
|
||||
Text: "MCH1211 received by procedure ORDPOST. (C D I R)", Age: 47 * time.Minute}}
|
||||
s.Unanswered = 1
|
||||
case "prod2":
|
||||
s.Subsys[len(s.Subsys)-1].Status, s.Subsys[len(s.Subsys)-1].Jobs = "INACTIVE", 0 // ORDERS: ended, not started again
|
||||
s.Ifaces[1].Status = "INACTIVE" // its backup line, set to start
|
||||
s.JobQueues = []ibmi.JobQueue{{Library: "ORDLIB", Name: "ORDERSQ", Waiting: 14}} // ORDERS isn't there to take them
|
||||
case "ha1":
|
||||
s.Subsys = s.Subsys[:len(s.Subsys)-1] // ORDERS runs on the active node only
|
||||
sbs := func(lib, name, status string, jobs int) ibmi.Subsystem {
|
||||
return ibmi.Subsystem{Library: lib, Name: name, Status: status, Jobs: jobs, Controlling: name == "QCTL"}
|
||||
}
|
||||
s.Subsys = []ibmi.Subsystem{sbs("QSYS", "QCTL", "ACTIVE", 3), sbs("QSYS", "QINTER", "ACTIVE", 20+k*7), sbs("QSYS", "QBATCH", "ACTIVE", 6+k),
|
||||
sbs("QSYS", "QUSRWRK", "ACTIVE", 60+k*9), sbs("QSYS", "QSERVER", "ACTIVE", 14), sbs("QSYS", "QSYSWRK", "ACTIVE", 110), sbs("QSYS", "QSPL", "ACTIVE", 4),
|
||||
sbs("QSYS", "QHTTPSVR", "ACTIVE", 9), sbs("QSYS", "QSNADS", "INACTIVE", 0), sbs("QSYS", "QCMN", "ACTIVE", 2), sbs("ORDLIB", "ORDERS", "ACTIVE", 12)}
|
||||
s.Ifaces = []ibmi.Interface{{Address: fmt.Sprintf("10.0.4.%d", 20+k), Prefix: 24, Line: "ETHLINE", Status: "ACTIVE", Autostart: true},
|
||||
{Address: fmt.Sprintf("10.0.9.%d", 20+k), Prefix: 24, Line: "ETHBKP", Status: "ACTIVE", Autostart: true}}
|
||||
switch {
|
||||
case part == "prod1" && bad:
|
||||
s.Inquiries = []ibmi.Message{{Key: "0001A2F0", ID: "CPA0701", Inquiry: true, Severity: 99, Job: "418276/BATCH/NIGHTLY", User: "BATCH",
|
||||
Text: "MCH1211 received by procedure ORDPOST. (C D I R)", Age: 7 * time.Minute}}
|
||||
s.Unanswered = 1
|
||||
case part == "prod2" && bad:
|
||||
s.Subsys[len(s.Subsys)-1].Status, s.Subsys[len(s.Subsys)-1].Jobs = "INACTIVE", 0 // ORDERS: ended, not started again
|
||||
s.JobQueues = []ibmi.JobQueue{{Library: "ORDLIB", Name: "ORDERSQ", Waiting: 14}} // ORDERS isn't there to take them
|
||||
case part == "ha1":
|
||||
s.Subsys = s.Subsys[:len(s.Subsys)-1] // ORDERS runs on the active node only
|
||||
}
|
||||
return s
|
||||
}
|
||||
now := time.Now()
|
||||
for _, r := range ibmi.Records(s) {
|
||||
r.Since = now.Add(-time.Duration(40+k*7) * 24 * time.Hour)
|
||||
recs := func(bad bool) []report.Workload {
|
||||
out := ibmi.Records(snap(bad))
|
||||
for i := range out {
|
||||
out[i].Since = now.Add(-time.Duration(40+k*7) * 24 * time.Hour)
|
||||
}
|
||||
return out
|
||||
}
|
||||
for _, r := range recs(false) {
|
||||
c.recs[r.ID] = r
|
||||
}
|
||||
switch part {
|
||||
case "prod1":
|
||||
c.flapSet(spellInquiry, recs(false), recs(true))
|
||||
case "prod2":
|
||||
c.flapSet(spellOrdersDown, recs(false), recs(true))
|
||||
}
|
||||
return c
|
||||
}
|
||||
@@ -589,8 +589,8 @@ const (
|
||||
|
||||
var vmRoles = [libvirtVMs]string{"db", "web", "cache", "build", "mq", "ldap", "backup", "monitor", "runner", "jump", "proxy", "spare"}
|
||||
|
||||
// hypervisorVMs are host i's domains: all running, but its spare is shut
|
||||
// off, and on the third hypervisor one has crashed.
|
||||
// hypervisorVMs are host i's domains: all running but its spare, which is
|
||||
// shut off.
|
||||
func hypervisorVMs(i int, now time.Time) []report.Workload {
|
||||
out := make([]report.Workload, libvirtVMs)
|
||||
for k := range out {
|
||||
@@ -598,11 +598,8 @@ func hypervisorVMs(i int, now time.Time) []report.Workload {
|
||||
r := report.Workload{Kind: report.KindVM, ID: id, Name: fmt.Sprintf("%s-%02d", vmRoles[k], i/hypervisorEvery),
|
||||
State: report.StateOK, Word: "Running", Since: now.Add(-time.Duration(3+k) * 24 * time.Hour).Truncate(time.Second),
|
||||
Facts: map[string]any{"firmware_uuid": id, "autostart": k != libvirtVMs-1, "hypervisor": "libvirt", "vcpus": 2 << (k % 3), "memory_gib": float64(int(4) << (k % 3))}}
|
||||
switch {
|
||||
case k == libvirtVMs-1:
|
||||
if k == libvirtVMs-1 {
|
||||
r.State, r.Word = report.StateStopped, "Shut off"
|
||||
case i/hypervisorEvery == 2 && k == 7:
|
||||
r.State, r.Word, r.Since = report.StateDown, "Crashed", now.Add(-47*time.Minute).Truncate(time.Second)
|
||||
}
|
||||
out[k] = r
|
||||
}
|
||||
|
||||
@@ -35,21 +35,12 @@ func newNutanix(name, token string, vms int) *fakeCluster {
|
||||
r := report.Workload{Kind: report.KindNode, ID: fmt.Sprintf("a1b2c3d4-%04d-4000-8000-%012d", cl, n), Name: host(cl, n),
|
||||
State: report.StateOK, Word: "Connected", Since: now.Add(-time.Duration(20+n) * 24 * time.Hour),
|
||||
Facts: map[string]any{"cluster": peName(cl), "cpus": 64, "memory_gib": 1024.0, "version": "AHV 10.0.1", "model": "NX-8170-G9"}}
|
||||
switch {
|
||||
case cl == 1 && n == 4:
|
||||
r.State, r.Word, r.Since = report.StateUpdating, "In maintenance mode", now.Add(-20*time.Minute)
|
||||
case cl == 2 && n == 2:
|
||||
r.State, r.Word, r.Facts["alerts"] = report.StateDegraded, "Alert from Prism: Disk Failure", "Disk Failure (critical)"
|
||||
}
|
||||
put(r)
|
||||
}
|
||||
for k, ct := range []string{"default-container", "vdi-images", "db-data"} {
|
||||
r := report.Workload{Kind: report.KindPVC, ID: fmt.Sprintf("c0ffee00-%04d-4000-8000-%012d", cl, k), Name: ct,
|
||||
State: report.StateOK, Word: "Available", Since: now.Add(-200 * 24 * time.Hour),
|
||||
Facts: map[string]any{"type": "storage container", "cluster": peName(cl), "capacity_gib": 81920.0, "replication_factor": 2}}
|
||||
if cl == 0 && k == 2 {
|
||||
r.State, r.Word, r.Facts["alerts"] = report.StateDegraded, "Alert from Prism: Storage Container Space Usage Exceeded", "Storage Container Space Usage Exceeded (critical)"
|
||||
}
|
||||
put(r)
|
||||
}
|
||||
}
|
||||
@@ -72,5 +63,8 @@ func newNutanix(name, token string, vms int) *fakeCluster {
|
||||
}
|
||||
put(r)
|
||||
}
|
||||
c.flap(fmt.Sprintf("a1b2c3d4-%04d-4000-8000-%012d", 1, 4), spellAHVUpdate, func(r *report.Workload, since time.Time) { // an AOS upgrade's rolling restart (scenario.go)
|
||||
r.State, r.Word, r.Since = report.StateUpdating, "In maintenance mode", since
|
||||
})
|
||||
return c
|
||||
}
|
||||
@@ -45,9 +45,6 @@ func newProxmox(name, token string, guests int) *fakeCluster {
|
||||
switch {
|
||||
case k%23 == 22:
|
||||
r.State, r.Word, r.Facts["ha"] = report.StateStopped, "Stopped", "stopped"
|
||||
case k == 41:
|
||||
r.State, r.Word, r.Facts["ha"] = report.StateDown, "HA can't run it (error)", "error"
|
||||
r.Since = now.Add(-18 * time.Minute)
|
||||
case k == 57:
|
||||
r.State, r.Word, r.Facts["lock"] = report.StateUpdating, "Migrating", "migrate"
|
||||
case k%50 == 9:
|
||||
@@ -55,5 +52,8 @@ func newProxmox(name, token string, guests int) *fakeCluster {
|
||||
}
|
||||
put(r)
|
||||
}
|
||||
c.flap("lxc/141", spellPVEHA, func(r *report.Workload, since time.Time) { // scenario.go
|
||||
r.State, r.Word, r.Since, r.Facts["ha"] = report.StateDown, "HA can't run it (error)", since, "error"
|
||||
})
|
||||
return c
|
||||
}
|
||||
+27
-23
@@ -27,25 +27,21 @@ func newPure(name, token string, k int) *fakeCluster {
|
||||
}
|
||||
|
||||
// newFlashBlade is fb-prod-01 (docs/pure.md, step 5): ten blades, one
|
||||
// evacuating; a failed fan; 400 file systems, cluster-02's first 100 VM
|
||||
// disks among them (Portworx names them after their PersistentVolumes),
|
||||
// one whose replication to fb-dr-01 is unhealthy; 40 buckets.
|
||||
// evacuating now and then (scenario.go); 400 file systems, cluster-02's
|
||||
// first 100 VM disks among them (Portworx names them after their
|
||||
// PersistentVolumes); 40 buckets.
|
||||
func newFlashBlade(name, token string, k int) *fakeCluster {
|
||||
arr := pureArrays[k]
|
||||
c := &fakeCluster{name: name, token: token, recs: map[string]report.Workload{}, changed: map[string]bool{}, noUsage: true, source: arr}
|
||||
c.usageFn = func(now time.Time) []report.Usage { return pureIO(k, now) }
|
||||
now := time.Now()
|
||||
put := func(w report.Workload) { c.recs[w.ID] = w }
|
||||
put(report.Workload{Kind: report.KindCluster, ID: report.ClusterSelf, Name: arr, State: report.StateDegraded, Word: "Hardware: CH1.FAN2 critical (Fan speed is 0 RPM)",
|
||||
Since: now.Add(-95 * time.Minute), Facts: map[string]any{"distribution": "Purity//FB", "version": "4.5.6", "model": "FlashBlade//S", "capacity_tib": 2000.0,
|
||||
"used_pct": 61.0, "data_reduction": 3.1, "hardware": "CH1.FAN2 critical (Fan speed is 0 RPM)", "alerts": "CH1.FAN2: Fan failure (critical)"}})
|
||||
put(report.Workload{Kind: report.KindCluster, ID: report.ClusterSelf, Name: arr, State: report.StateOK, Word: "API answering",
|
||||
Since: now.Add(-200 * 24 * time.Hour), Facts: map[string]any{"distribution": "Purity//FB", "version": "4.5.6", "model": "FlashBlade//S", "capacity_tib": 2000.0,
|
||||
"used_pct": 61.0, "data_reduction": 3.1}})
|
||||
for i := 1; i <= 10; i++ {
|
||||
r := report.Workload{Kind: report.KindNode, ID: fmt.Sprintf("ch1.fb%d", i), Name: fmt.Sprintf("CH1.FB%d", i), State: report.StateOK, Word: "Healthy",
|
||||
Since: now.Add(-200 * 24 * time.Hour), Facts: map[string]any{"raw_tib": 48.0}}
|
||||
if i == 6 {
|
||||
r.State, r.Word, r.Since = report.StateUpdating, "Evacuating its data (37%)", now.Add(-2*time.Hour)
|
||||
}
|
||||
put(r)
|
||||
put(report.Workload{Kind: report.KindNode, ID: fmt.Sprintf("ch1.fb%d", i), Name: fmt.Sprintf("CH1.FB%d", i), State: report.StateOK, Word: "Healthy",
|
||||
Since: now.Add(-200 * 24 * time.Hour), Facts: map[string]any{"raw_tib": 48.0}})
|
||||
}
|
||||
put(report.Workload{Kind: report.KindCluster, ID: "connection/c-fb-dr-01", Name: "connection/fb-dr-01", State: report.StateOK, Word: "Connected",
|
||||
Since: now.Add(-90 * 24 * time.Hour), Facts: map[string]any{"type": "async-replication", "version": "4.5.6"}})
|
||||
@@ -62,9 +58,7 @@ func newFlashBlade(name, token string, k int) *fakeCluster {
|
||||
r.Name, r.Word, r.Facts["protocols"] = "home", "Shared over NFSv3, NFSv4.1, SMB", "NFSv3, NFSv4.1, SMB"
|
||||
r.Facts["replication"] = "to fb-dr-01, replicating, 4 min behind"
|
||||
case n == 101:
|
||||
r.Name, r.State = "scratch", report.StateDegraded
|
||||
r.Word = "Replication to fb-dr-01 is unhealthy: Target file system is full"
|
||||
r.Facts["replication"] = "to fb-dr-01, unhealthy, 5 h behind"
|
||||
r.Name = "scratch"
|
||||
case n%50 == 7:
|
||||
r.Word = "Not shared over any protocol"
|
||||
delete(r.Facts, "protocols")
|
||||
@@ -80,6 +74,10 @@ func newFlashBlade(name, token string, k int) *fakeCluster {
|
||||
}
|
||||
put(r)
|
||||
}
|
||||
c.flap("ch1.fb6", spellBladeEvac, func(r *report.Workload, since time.Time) {
|
||||
done := min(95, 5+int(time.Since(since).Minutes()))
|
||||
r.State, r.Word, r.Since = report.StateUpdating, fmt.Sprintf("Evacuating its data (%d%%)", done), since
|
||||
})
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -99,12 +97,8 @@ func newFlashArray(name, token string, k, volumes int) *fakeCluster {
|
||||
}
|
||||
put(self)
|
||||
for n, mode := range []string{"primary", "secondary"} {
|
||||
r := report.Workload{Kind: report.KindNode, ID: fmt.Sprintf("ct%d", n), Name: fmt.Sprintf("CT%d", n), State: report.StateOK, Word: "Ready, " + mode,
|
||||
Since: now.Add(-40 * 24 * time.Hour), Facts: map[string]any{"mode": mode, "model": "FA-X70R4", "version": "6.7.3"}}
|
||||
if arr == "fa-vdi-01" && n == 1 {
|
||||
r.State, r.Word, r.Since = report.StateUpdating, "Updating", now.Add(-12*time.Minute)
|
||||
}
|
||||
put(r)
|
||||
put(report.Workload{Kind: report.KindNode, ID: fmt.Sprintf("ct%d", n), Name: fmt.Sprintf("CT%d", n), State: report.StateOK, Word: "Ready, " + mode,
|
||||
Since: now.Add(-40 * 24 * time.Hour), Facts: map[string]any{"mode": mode, "model": "FA-X70R4", "version": "6.7.3"}})
|
||||
}
|
||||
switch arr {
|
||||
case "fa-prod-01", "fa-prod-02":
|
||||
@@ -116,8 +110,8 @@ func newFlashArray(name, token string, k, volumes int) *fakeCluster {
|
||||
put(report.Workload{Kind: report.KindCluster, ID: "connection/c-fa-dr-01", Name: "connection/fa-dr-01", State: report.StateOK, Word: "Connected",
|
||||
Since: now.Add(-60 * 24 * time.Hour), Facts: map[string]any{"type": "async-replication", "version": "6.7.3"}})
|
||||
case "fa-dr-01":
|
||||
put(report.Workload{Kind: report.KindCluster, ID: "connection/c-fa-prod-01", Name: "connection/fa-prod-01", State: report.StateDegraded,
|
||||
Word: "Connected, but its paths aren't redundant", Since: now.Add(-50 * time.Minute), Facts: map[string]any{"type": "async-replication", "version": "6.7.3"}})
|
||||
put(report.Workload{Kind: report.KindCluster, ID: "connection/c-fa-prod-01", Name: "connection/fa-prod-01", State: report.StateOK,
|
||||
Word: "Connected", Since: now.Add(-60 * 24 * time.Hour), Facts: map[string]any{"type": "async-replication", "version": "6.7.3"}})
|
||||
}
|
||||
prefix := map[string]string{"fa-prod-01": "k8s-pvc", "fa-prod-02": "k8s-pvc", "fa-dr-01": "dr-copy", "fa-vdi-01": "vdi"}[arr]
|
||||
for n := range volumes {
|
||||
@@ -146,6 +140,16 @@ func newFlashArray(name, token string, k, volumes int) *fakeCluster {
|
||||
}
|
||||
put(r)
|
||||
}
|
||||
switch arr {
|
||||
case "fa-vdi-01":
|
||||
c.flap("ct1", spellVDIUpdate, func(r *report.Workload, since time.Time) {
|
||||
r.State, r.Word, r.Since = report.StateUpdating, "Updating", since
|
||||
})
|
||||
case "fa-dr-01":
|
||||
c.flap("connection/c-fa-prod-01", spellDRPaths, func(r *report.Workload, since time.Time) {
|
||||
r.State, r.Word, r.Since = report.StateDegraded, "Connected, but its paths aren't redundant", since
|
||||
})
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
package main
|
||||
|
||||
import "time"
|
||||
|
||||
// What goes wrong in the demo, and when (#35). A real fleet is mostly
|
||||
// fine: a few things stay wrong until someone gets to them, and the rest
|
||||
// come and go. So the demo has
|
||||
//
|
||||
// - three stories, each a cause with its effects:
|
||||
// 1. fa-prod-01 lost a drive: the array is degraded and its writes are
|
||||
// slow while the drive's data is rebuilt (all the time);
|
||||
// 2. checkout errors: every 2 hours fleet-0032, a PostgreSQL replica,
|
||||
// falls behind, HAProxy takes it out, and the shop's checkout answers
|
||||
// with errors until it catches up (20 minutes);
|
||||
// 3. the legacy datacenter: every 4 hours its server stops answering
|
||||
// Akamai's liveness test, GTM has nowhere to send the legacy property,
|
||||
// and its CP code fails (30 minutes);
|
||||
// - a few lasting warnings anyone would have: a failover nobody moved
|
||||
// back, a weak key, a VM snapshot left behind, a datastore filling, a
|
||||
// zone on Flexible SSL, a renewal waiting on someone;
|
||||
// - problems that come and go: a unit that fails and is restarted, a
|
||||
// container that exits, a check that flaps, a host update that takes
|
||||
// a while, a late backup, an IBM i job waiting on an answer.
|
||||
//
|
||||
// Spells count from the Unix epoch, not from when the simulator started,
|
||||
// so every part of it agrees on what's happening now, a restarted demo
|
||||
// picks up where the clock is, and the history the demo starts with can
|
||||
// be worked out for any time in the past.
|
||||
|
||||
// spell is when one problem is happening: for on, once every every, at
|
||||
// into each period. A zero every means all the time.
|
||||
type spell struct {
|
||||
every, at, on time.Duration
|
||||
}
|
||||
|
||||
// always is a problem that's there all the time.
|
||||
var always = spell{}
|
||||
|
||||
func (s spell) active(t time.Time) bool {
|
||||
if s.every == 0 {
|
||||
return true
|
||||
}
|
||||
p := time.Duration(t.UnixNano()) % s.every
|
||||
return p >= s.at && p < s.at+s.on
|
||||
}
|
||||
|
||||
// began is when the current spell started, or the last one if it's over;
|
||||
// a long time ago for a problem that's always there.
|
||||
func (s spell) began(t time.Time) time.Time {
|
||||
if s.every == 0 {
|
||||
return t.Add(-9 * 24 * time.Hour)
|
||||
}
|
||||
p := time.Duration(t.UnixNano()) % s.every
|
||||
start := t.Add(-p + s.at)
|
||||
if start.After(t) {
|
||||
start = start.Add(-s.every)
|
||||
}
|
||||
return start
|
||||
}
|
||||
|
||||
// The spells, by story. Periods aren't multiples of each other, so the
|
||||
// mix on screen changes through the day.
|
||||
var (
|
||||
// Story 2: checkout errors.
|
||||
spellReplicaBehind = spell{every: 2 * time.Hour, at: 40 * time.Minute, on: 20 * time.Minute}
|
||||
// Story 3: the legacy datacenter.
|
||||
spellLegacyDC = spell{every: 4 * time.Hour, at: 70 * time.Minute, on: 30 * time.Minute}
|
||||
|
||||
// Coming and going.
|
||||
spellRedisFails = spell{every: 110 * time.Minute, at: 20 * time.Minute, on: 12 * time.Minute}
|
||||
spellAppExits = spell{every: 75 * time.Minute, at: 5 * time.Minute, on: 8 * time.Minute}
|
||||
spellLegacy502 = spell{every: 140 * time.Minute, at: 30 * time.Minute, on: 15 * time.Minute}
|
||||
spellLBFlaps = spell{every: 100 * time.Minute, at: 60 * time.Minute, on: 15 * time.Minute}
|
||||
spellCrashloop = spell{every: 150 * time.Minute, at: 15 * time.Minute, on: 20 * time.Minute}
|
||||
spellESXiUpdate = spell{every: 4 * time.Hour, at: 10 * time.Minute, on: 40 * time.Minute}
|
||||
spellESXiMemory = spell{every: 130 * time.Minute, at: 50 * time.Minute, on: 15 * time.Minute}
|
||||
spellAHVUpdate = spell{every: 270 * time.Minute, at: 100 * time.Minute, on: 40 * time.Minute}
|
||||
spellPVEHA = spell{every: 150 * time.Minute, at: 90 * time.Minute, on: 20 * time.Minute}
|
||||
spellDRPaths = spell{every: 3 * time.Hour, at: 130 * time.Minute, on: 30 * time.Minute}
|
||||
spellBladeEvac = spell{every: 6 * time.Hour, at: 2 * time.Hour, on: 60 * time.Minute}
|
||||
spellVDIUpdate = spell{every: 5 * time.Hour, at: 4 * time.Hour, on: 45 * time.Minute}
|
||||
spellInquiry = spell{every: 3 * time.Hour, at: 110 * time.Minute, on: 30 * time.Minute}
|
||||
spellOrdersDown = spell{every: 200 * time.Minute, at: 80 * time.Minute, on: 25 * time.Minute}
|
||||
spellVaultEdgeDown = spell{every: 3 * time.Hour, at: 150 * time.Minute, on: 20 * time.Minute}
|
||||
spellBlocklisted = spell{every: 5 * time.Hour, at: 3 * time.Hour, on: 40 * time.Minute}
|
||||
spellLateBackup = spell{every: 6 * time.Hour, at: 5 * time.Hour, on: 50 * time.Minute}
|
||||
spellBackupShrank = spell{every: 6 * time.Hour, at: 2 * time.Hour, on: 50 * time.Minute}
|
||||
spellSQLAgent = spell{every: 4 * time.Hour, at: 3 * time.Hour, on: 45 * time.Minute}
|
||||
spellFSCrash = spell{every: 8 * time.Hour, at: 7 * time.Hour, on: time.Hour}
|
||||
spellActivation = spell{every: 3 * time.Hour, at: 20 * time.Minute, on: 80 * time.Minute}
|
||||
)
|
||||
@@ -0,0 +1,106 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/cairnobs/cairnobs/internal/report"
|
||||
)
|
||||
|
||||
func TestSpell(t *testing.T) {
|
||||
s := spell{every: time.Hour, at: 20 * time.Minute, on: 10 * time.Minute}
|
||||
day := time.Date(2026, 11, 16, 0, 0, 0, 0, time.UTC)
|
||||
for _, c := range []struct {
|
||||
at time.Duration
|
||||
active bool
|
||||
began time.Duration
|
||||
}{
|
||||
{19 * time.Minute, false, -40 * time.Minute},
|
||||
{20 * time.Minute, true, 20 * time.Minute},
|
||||
{29 * time.Minute, true, 20 * time.Minute},
|
||||
{30 * time.Minute, false, 20 * time.Minute},
|
||||
{80 * time.Minute, true, 80 * time.Minute},
|
||||
} {
|
||||
now := day.Add(c.at)
|
||||
if s.active(now) != c.active || !s.began(now).Equal(day.Add(c.began)) {
|
||||
t.Errorf("at %s: active %v began %s", c.at, s.active(now), s.began(now).Sub(day))
|
||||
}
|
||||
}
|
||||
if !always.active(day) || !always.began(day).Before(day) {
|
||||
t.Error("always")
|
||||
}
|
||||
}
|
||||
|
||||
// The demo stays calm: across a day, no more than a few of the problems
|
||||
// that come and go are on at once, and every one of them is on some of
|
||||
// the time and off the rest.
|
||||
func TestScenarioIsCalm(t *testing.T) {
|
||||
spells := map[string]spell{"replica": spellReplicaBehind, "legacy dc": spellLegacyDC, "redis": spellRedisFails, "app": spellAppExits,
|
||||
"502": spellLegacy502, "lb": spellLBFlaps, "crashloop": spellCrashloop, "esxi update": spellESXiUpdate, "esxi memory": spellESXiMemory,
|
||||
"ahv": spellAHVUpdate, "pve": spellPVEHA, "dr paths": spellDRPaths, "blade": spellBladeEvac, "vdi": spellVDIUpdate,
|
||||
"inquiry": spellInquiry, "orders": spellOrdersDown, "vault edge": spellVaultEdgeDown, "blocklist": spellBlocklisted,
|
||||
"late backup": spellLateBackup, "shrank": spellBackupShrank, "sql agent": spellSQLAgent, "fs crash": spellFSCrash, "activation": spellActivation}
|
||||
on := map[string]int{}
|
||||
most, total, steps := 0, 0, 0
|
||||
day := time.Date(2026, 11, 16, 0, 0, 0, 0, time.UTC)
|
||||
for m := 0; m < 24*60; m++ {
|
||||
now := day.Add(time.Duration(m) * time.Minute)
|
||||
n := 0
|
||||
for name, s := range spells {
|
||||
if s.on <= 0 || s.at+s.on > s.every {
|
||||
t.Fatalf("%s: on %s at %s of %s", name, s.on, s.at, s.every)
|
||||
}
|
||||
if s.active(now) {
|
||||
on[name]++
|
||||
n++
|
||||
}
|
||||
}
|
||||
most, total, steps = max(most, n), total+n, steps+1
|
||||
}
|
||||
for name := range spells {
|
||||
if on[name] == 0 || on[name] == 24*60 {
|
||||
t.Errorf("%s is on %d minutes of the day", name, on[name])
|
||||
}
|
||||
}
|
||||
if avg := float64(total) / float64(steps); avg > 4 || most > 9 {
|
||||
t.Errorf("on at once: %.1f on average, %d at most", avg, most)
|
||||
}
|
||||
}
|
||||
|
||||
// A flapping record goes wrong during its spell and comes right after; a
|
||||
// record only the bad snapshot has comes and goes.
|
||||
func TestFlapSet(t *testing.T) {
|
||||
c := &fakeCluster{recs: map[string]report.Workload{}, changed: map[string]bool{}}
|
||||
good := []report.Workload{{ID: "a", State: report.StateOK, Word: "Active"}, {ID: "b", State: report.StateOK, Word: "Fine"}}
|
||||
bad := []report.Workload{{ID: "a", State: report.StateDown, Word: "Inactive"}, {ID: "b", State: report.StateOK, Word: "Fine"},
|
||||
{ID: "q", State: report.StateDegraded, Word: "14 jobs waiting"}}
|
||||
for _, r := range good {
|
||||
c.recs[r.ID] = r
|
||||
}
|
||||
s := spell{every: time.Hour, at: 0, on: 30 * time.Minute}
|
||||
c.flapSet(s, good, bad)
|
||||
if len(c.episodes) != 2 {
|
||||
t.Fatalf("%d episodes, want a and q", len(c.episodes))
|
||||
}
|
||||
hour := time.Date(2026, 11, 16, 3, 0, 0, 0, time.UTC)
|
||||
c.play(hour.Add(5 * time.Minute))
|
||||
if c.recs["a"].State != report.StateDown || c.recs["q"].ID != "q" {
|
||||
t.Fatalf("in the spell: %+v %+v", c.recs["a"], c.recs["q"])
|
||||
}
|
||||
c.removed = nil
|
||||
c.play(hour.Add(40 * time.Minute))
|
||||
if _, there := c.recs["q"]; c.recs["a"].State != report.StateOK || there || len(c.removed) != 1 {
|
||||
t.Fatalf("after: %+v, q there %v, removed %v", c.recs["a"], there, c.removed)
|
||||
}
|
||||
}
|
||||
|
||||
// IBM i's flapping is only what the bad snapshot changes, not the whole
|
||||
// partition.
|
||||
func TestIBMiFlapsLittle(t *testing.T) {
|
||||
for k, part := range ibmiPartitions[:2] {
|
||||
c := newIBMi("fleet-0041", "t", k)
|
||||
if n := len(c.episodes); n == 0 || n > 4 {
|
||||
t.Errorf("%s: %d records flap", part, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
+15
-13
@@ -31,9 +31,11 @@ import (
|
||||
// - vault-prod: Vault Enterprise, three healthy Raft servers, DR
|
||||
// replication streaming to its secondary, a license 300 days out, an
|
||||
// audit device, and a PKI mount (pki_int) with 400 services'
|
||||
// certificates (some ending soon) and 2,000 short-lived ones.
|
||||
// - vault-edge: OpenBao, three Raft servers with one failed (no failure
|
||||
// tolerance left), and an intermediate CA 12 days from its end.
|
||||
// certificates (one ending within a week, its renewal overdue) and
|
||||
// 2,000 short-lived ones.
|
||||
// - vault-edge: OpenBao, three Raft servers, one of which fails now and
|
||||
// then and leaves no failure tolerance (scenario.go), and an
|
||||
// intermediate CA 12 days from its end.
|
||||
//
|
||||
// They listen on vaultPort and the port after it; -vault's directory
|
||||
// keeps their certificate and key, so vault.yaml (written there) stays
|
||||
@@ -74,7 +76,7 @@ func serveVaults(ctx context.Context, dir string) {
|
||||
newFakeVault("vault-prod", "1.21.4+ent", true, "pki_int", now, 300*24*time.Hour, 400, 2000,
|
||||
[]fakeRaft{{"v1", "leader", "alive"}, {"v2", "voter", "alive"}, {"v3", "voter", "alive"}}, 1, 18_000),
|
||||
newFakeVault("vault-edge", "2.4.1", false, "pki_edge", now, 12*24*time.Hour, 40, 0,
|
||||
[]fakeRaft{{"e1", "leader", "alive"}, {"e2", "voter", "alive"}, {"e3", "voter", "failed"}}, 0, 900),
|
||||
[]fakeRaft{{"e1", "leader", "alive"}, {"e2", "voter", "alive"}, {"e3", "voter", "alive"}}, 1, 900),
|
||||
}
|
||||
var yaml strings.Builder
|
||||
yaml.WriteString("vault:\n")
|
||||
@@ -153,14 +155,11 @@ func newFakeVault(name, version string, ent bool, mount string, now time.Time, c
|
||||
day := 24 * time.Hour
|
||||
for i := range services {
|
||||
cn := fmt.Sprintf("svc-%03d.%s.internal", i, strings.TrimPrefix(name, "vault-"))
|
||||
// Most a long way from their end; a few within the warning's 21
|
||||
// days, fewer within the critical 7.
|
||||
// Most a long way from their end; one of vault-prod's within a
|
||||
// week, its renewal overdue.
|
||||
left := time.Duration(25+(i*37)%340) * day
|
||||
switch {
|
||||
case i%97 == 5:
|
||||
left = 5 * day
|
||||
case i%41 == 7:
|
||||
left = 15 * day
|
||||
if name == "vault-prod" && i == 102 {
|
||||
left = 6 * day
|
||||
}
|
||||
for k, end := range []time.Duration{left - 30*day, left} { // the one it replaced, then the renewal
|
||||
x, p := sign(&x509.Certificate{Subject: pkix.Name{CommonName: cn}, DNSNames: []string{cn}, NotBefore: now.Add(end - 365*day), NotAfter: now.Add(end)},
|
||||
@@ -209,8 +208,11 @@ func (v *fakeVault) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
case p == "sys/leader":
|
||||
out = map[string]any{"ha_enabled": true, "is_self": true, "leader_address": "https://" + v.servers[0].id + "." + v.name + ":8200"}
|
||||
case p == "sys/storage/raft/autopilot/state":
|
||||
servers, voters, healthy := map[string]any{}, []string{}, true
|
||||
servers, voters, healthy, ft := map[string]any{}, []string{}, true, v.ft
|
||||
for _, s := range v.servers {
|
||||
if v.name == "vault-edge" && s.id == "e3" && spellVaultEdgeDown.active(now) {
|
||||
s.node, s.status, ft = "failed", "voter", 0
|
||||
}
|
||||
ok := s.node == "alive"
|
||||
healthy = healthy && ok
|
||||
sv := map[string]any{"id": s.id, "name": v.name + "-" + s.id, "address": s.id + ":8201", "node_status": s.node, "healthy": ok,
|
||||
@@ -221,7 +223,7 @@ func (v *fakeVault) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
servers[s.id] = sv
|
||||
voters = append(voters, s.id)
|
||||
}
|
||||
out = data(map[string]any{"healthy": healthy, "failure_tolerance": v.ft, "voters": voters, "servers": servers})
|
||||
out = data(map[string]any{"healthy": healthy, "failure_tolerance": ft, "voters": voters, "servers": servers})
|
||||
case p == "auth/token/lookup-self":
|
||||
out = data(map[string]any{"display_name": "token-obs-beacond", "policies": []string{"default", "obs-monitor"}, "ttl": 20 * 3600,
|
||||
"creation_ttl": 24 * 3600, "renewable": true, "type": "service"})
|
||||
|
||||
@@ -30,12 +30,6 @@ func newVSphere(name, token string, vms int) *fakeCluster {
|
||||
r := report.Workload{Kind: report.KindNode, ID: fmt.Sprintf("host-%d", 1000+n), Name: fmt.Sprintf("esxi-%02d.example.org", n),
|
||||
State: report.StateOK, Word: "Connected", Since: now.Add(-time.Duration(30+n) * 24 * time.Hour),
|
||||
Facts: map[string]any{"cpus": 48, "memory_gib": 768.0, "version": "VMware ESXi 8.0.3 build-24414501"}}
|
||||
switch n {
|
||||
case 9:
|
||||
r.State, r.Word, r.Since = report.StateUpdating, "In maintenance mode", now.Add(-35*time.Minute)
|
||||
case 13:
|
||||
r.State, r.Word, r.Facts["alarms"] = report.StateDegraded, "vCenter alarm: Host memory usage", "Host memory usage (red)"
|
||||
}
|
||||
put(r)
|
||||
}
|
||||
for d := 1; d <= 6; d++ {
|
||||
@@ -62,13 +56,18 @@ func newVSphere(name, token string, vms int) *fakeCluster {
|
||||
r.State, r.Word = report.StateStopped, "Powered off"
|
||||
case k == 77:
|
||||
r.State, r.Word, r.Facts["oldest_snapshot"] = report.StateDegraded, "Running, with a snapshot 23 days old", now.Add(-23*24*time.Hour).Format(time.DateOnly)
|
||||
case k == 150:
|
||||
r.State, r.Word = report.StateDown, "Inaccessible: its files can't be reached"
|
||||
r.Since = now.Add(-9 * time.Minute)
|
||||
case k%40 == 5:
|
||||
r.Facts["vmware_tools"] = "not running"
|
||||
}
|
||||
put(r)
|
||||
}
|
||||
// Patching takes a host into maintenance for a while; a busy one
|
||||
// raises a memory alarm now and then (scenario.go).
|
||||
c.flap("host-1009", spellESXiUpdate, func(r *report.Workload, since time.Time) {
|
||||
r.State, r.Word, r.Since = report.StateUpdating, "In maintenance mode", since
|
||||
})
|
||||
c.flap("host-1013", spellESXiMemory, func(r *report.Workload, since time.Time) {
|
||||
r.State, r.Word, r.Since, r.Facts["alarms"] = report.StateDegraded, "vCenter alarm: Host memory usage", since, "Host memory usage (red)"
|
||||
})
|
||||
return c
|
||||
}
|
||||
+23
-12
@@ -14,10 +14,11 @@ import (
|
||||
// agents named win-*, outside the sites of 50:
|
||||
//
|
||||
// win-dc01 a domain controller, all well
|
||||
// win-fs01 a file server that rebooted itself 25 minutes ago (Kernel-Power
|
||||
// 41, critical) and has a disk with bad blocks
|
||||
// win-sql01 SQL Server: its Agent service should start by itself and has
|
||||
// stopped; its certificate ends in a few days
|
||||
// win-fs01 a file server with a disk going bad: bad blocks, and every
|
||||
// 8 hours it crashes and comes back (Kernel-Power 41, critical)
|
||||
// win-sql01 SQL Server: its Agent service should start by itself, and
|
||||
// now and then stops (scenario.go); its certificate ends in a
|
||||
// few days
|
||||
// win-hv01 a Hyper-V host with twelve VMs: one saved, one failed
|
||||
type winServer struct {
|
||||
name, os, kernel string
|
||||
@@ -73,17 +74,26 @@ func newWindowsHost(name string, interval time.Duration) *host {
|
||||
// them.
|
||||
func (wh *winHost) stamp(h *host, r *report.Report, now time.Time) {
|
||||
w := wh.w
|
||||
crashes, lastCrash := 0, time.Time{}
|
||||
if w.name == "win-fs01" {
|
||||
// It went down without shutting down cleanly 25 minutes ago, and
|
||||
// still has, whenever one looks.
|
||||
h.start = now.Add(-25 * time.Minute)
|
||||
r.Uptime = now.Sub(h.start).Seconds()
|
||||
// It goes down without shutting down cleanly and is straight back:
|
||||
// up since the last time.
|
||||
lastCrash = spellFSCrash.began(now)
|
||||
for t := lastCrash; now.Sub(t) < 24*time.Hour; t = t.Add(-spellFSCrash.every) {
|
||||
crashes++
|
||||
}
|
||||
h.start = lastCrash
|
||||
r.Uptime = max(0, now.Sub(h.start).Seconds())
|
||||
}
|
||||
r.Units = nil
|
||||
for _, u := range w.units {
|
||||
un := report.Unit{Name: u + ".service", Load: "loaded", Active: "active", Sub: "running", Since: h.start.UTC().Add(time.Minute).Truncate(time.Second)}
|
||||
if w.name == "win-sql01" && u == "SQLSERVERAGENT" {
|
||||
un.Active, un.Sub, un.Since = "failed", "dead", time.Time{}
|
||||
if b := spellSQLAgent.began(now); spellSQLAgent.active(now) {
|
||||
un.Active, un.Sub, un.Since = "failed", "dead", time.Time{}
|
||||
} else if b.After(h.start) { // someone started it again
|
||||
un.Since = b.Add(spellSQLAgent.on).UTC().Truncate(time.Second)
|
||||
}
|
||||
}
|
||||
r.Units = append(r.Units, un)
|
||||
}
|
||||
@@ -94,13 +104,14 @@ func (wh *winHost) stamp(h *host, r *report.Report, now time.Time) {
|
||||
}
|
||||
switch w.name {
|
||||
case "win-fs01":
|
||||
group("System", "Microsoft-Windows-Kernel-Power", 41, "critical", 1, 24*time.Minute, 24*time.Minute,
|
||||
first := lastCrash.Add(-time.Duration(crashes-1) * spellFSCrash.every)
|
||||
group("System", "Microsoft-Windows-Kernel-Power", 41, "critical", crashes, now.Sub(first), now.Sub(h.start),
|
||||
"The system has rebooted without cleanly shutting down first. This error could be caused if the system stopped responding, crashed, or lost power unexpectedly.")
|
||||
group("System", "disk", 7, "error", 14, 19*time.Hour, 2*time.Hour, `The device, \Device\Harddisk1\DR1, has a bad block.`)
|
||||
group("System", "NETLOGON", 5719, "error", 2, 23*time.Minute, 23*time.Minute,
|
||||
group("System", "NETLOGON", 5719, "error", 2*crashes, now.Sub(first), now.Sub(h.start),
|
||||
"This computer was not able to set up a secure session with a domain controller in domain CORP due to the following: There are currently no logon servers available to service the logon request.")
|
||||
case "win-sql01":
|
||||
group("System", "Service Control Manager", 7034, "error", 3, 6*time.Hour, 40*time.Minute,
|
||||
group("System", "Service Control Manager", 7034, "error", 3, 6*time.Hour, now.Sub(spellSQLAgent.began(now)),
|
||||
"The SQL Server Agent (MSSQLSERVER) service terminated unexpectedly. It has done this 3 time(s).")
|
||||
case "win-hv01":
|
||||
group("System", "Microsoft-Windows-Hyper-V-Worker", 18560, "critical", 1, 3*time.Hour, 3*time.Hour,
|
||||
|
||||
Reference in new issue
Block a user