Demo: a plan for what goes wrong, with problems that come and go #48

Merged
jcoffey-dev merged 1 commits from feature/demo-story into main 2026-11-16 05:44:17 +00:00
19 changed files with 640 additions and 230 deletions

No files matched your search

+10
View File
@@ -49,3 +49,13 @@ obs-beacond and the simulator, so nothing in it can reach anything else:
- The root is read-only and the state is in tmpfs, so restarting the
container is a fresh demo; restart it on a timer (weekly is plenty) so
history doesn't pile up.
## What a visitor sees
Most of the simulated fleet is healthy. What's wrong follows a plan
(`scripts/fleet/scenario.go`, and "The demo fleet" in development.md):
three stories with a cause and its effects, a few lasting warnings, and
problems that come and go through the day, so incidents open, resolve and
show up under Recently resolved. Because the plan runs on the clock, two
visitors an hour apart see different problems, and a restarted demo shows
the same ones the old one would have.
+14 -5
View File
@@ -67,11 +67,20 @@ send (ntfy, Discord, PagerDuty, a webhook and email) and lists it on
http://127.0.0.1:58450/. `fleet run -h` lists the rest: clusters, VMs,
Proxmox, vSphere, Nutanix and Pure arrays.
Some things are wrong on purpose, so every page has something to show: a
container exited, a unit failed, a stale backup, PostgreSQL's leader away
from home with a replica behind, an origin failing Cloudflare's health
check, a zone on Flexible SSL, and so on. The mail server is healthy, to
show what normal looks like.
Some things are wrong on purpose, so every page has something to show,
and most of the fleet is fine, as a real one is. `scripts/fleet/scenario.go`
has the plan: three stories with a cause and its effects (a FlashArray
lost a drive; a PostgreSQL replica falls behind and the shop's checkout
fails; the legacy datacenter drops out of Akamai GTM), a handful of
lasting warnings (a failover nobody moved back, a weak key, an old VM
snapshot, a zone on Flexible SSL, and so on), and problems that come and
go: a unit that fails and is restarted, a container that exits, a check
that flaps, a late backup, a host being patched. Each of those is a
*spell*: on for a while once every so often, counted from the Unix epoch,
so every part of the simulator agrees on what's happening now and a
restarted demo carries on where the clock is. Incidents open and resolve
through the day, and a dozen or two are open at any moment. The mail
server is healthy, to show what normal looks like.
## Dependencies and notices
+75 -40
View File
@@ -17,18 +17,19 @@ import (
// account on loopback, every signature checked. GTM's datacenters are
// the first sites and its servers their load balancers' public
// addresses (netmap.go, positions 1 and 2), so a failing one is a fleet
// host. Problems on purpose:
// host. Problems on purpose, the ones marked * coming and going
// (scenario.go):
//
// GTM site 2's first load balancer failing for api.shop; the
// legacy property with no live datacenter; corp's change
// propagating for 45 minutes
// CPS 180 enrollments: four ending within 21 days (one within
// 7), two third-party renewals waiting on a CSR
// Property 320 properties: a failed production activation, one
// Manager stuck on staging, two rolling out
// GTM * site 2's first load balancer failing for api.shop;
// * the legacy property with no live datacenter (story 3)
// CPS 180 enrollments: a third-party renewal waiting on a CSR,
// ending in 15 days
// Property 320 properties: a failed production activation;
// Manager * one rolling out for longer than it should
// Edge DNS 40 zones (one of 5,000 records); one changes an MX 20
// minutes after the start
// Traffic 320 CP codes; one at 8% 5xx, one at 30%
// Traffic 320 CP codes; * the legacy site's at 30% 5xx while its
// datacenter is down (story 3)
const akamaiPort = 58410
func serveAkamai(ctx context.Context, dir string) {
@@ -48,6 +49,18 @@ func serveAkamai(ctx context.Context, dir string) {
f := &akamai.Fake{Creds: c}
start := time.Now()
n := fleetAkamai(f)
go func() {
t := time.NewTicker(30 * time.Second)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case now := <-t.C:
akamaiStory(f, now)
}
}
}()
srv := &http.Server{Handler: f, ReadHeaderTimeout: 5 * time.Second}
go func() { _ = srv.Serve(ln) }()
go func() { <-ctx.Done(); _ = srv.Close() }()
@@ -90,35 +103,25 @@ func fleetAkamai(f *akamai.Fake) string {
switch name {
case "shop":
ps[0].Name = "api"
if t := ps[0].Targets[3100+2%sites]; t != nil {
t[lb(2%sites, 1)] = false
}
case "corp":
ps = append(ps, akamai.FakeProperty{Name: "legacy", Type: "failover", Targets: map[int]map[string]bool{3100: {"198.51.100.50": false}}})
ps = append(ps, akamai.FakeProperty{Name: "legacy", Type: "failover", Targets: map[int]map[string]bool{3100: {legacyServer: true}}})
}
props += len(ps)
fd := akamai.FakeDomain{Type: "full", Datacenters: dcs, Properties: ps}
if name == "corp" {
fd.Propagation, fd.Since = "PENDING", now.Add(-45*time.Minute)
}
f.SetDomain(name+".example.akadns.net", fd)
f.SetDomain(name+".example.akadns.net", akamai.FakeDomain{Type: "full", Datacenters: dcs, Properties: ps})
}
// CPS: ECDSA keys (fast to make); every tenth has an RSA one too.
for i := range 180 {
cn := fmt.Sprintf("app%03d.example.com", i)
left := time.Duration(25+(i*37)%300) * day
switch i {
case 3:
left = 5 * day
case 11, 47, 101:
left = time.Duration(10+i%9) * day
if i == 47 {
left = 15 * day
}
e := akamai.FakeEnrollment{CN: cn, SANs: []string{cn, "www." + cn}, Validation: "dv", Contract: "1-FLEET", Waiting: "-"}
issuer := "Let's Encrypt E7"
if i%6 == 0 {
e.Validation, issuer = "ov", "DigiCert Global G2 TLS RSA SHA256 2020 CA1"
}
if i == 47 || i == 120 {
if i == 47 {
e.Validation, e.Waiting, issuer = "third-party", "third-party-csr", "Sectigo RSA Domain Validation Secure Server CA"
}
e.Deployed = []string{akamai.FakePEM(issuer, e.SANs, true, now.Add(left))}
@@ -142,15 +145,11 @@ func fleetAkamai(f *akamai.Fake) string {
p.Latest, p.Staging = v+1, v+1
p.Activations = append(p.Activations, staged,
akamai.FakeActivation{Version: v + 1, Network: "PRODUCTION", Status: "FAILED", Submitted: now.Add(-3 * time.Hour), Note: "Origin moved to the new load balancers"})
case 9:
p.Latest = v + 1
p.Activations = append(p.Activations, akamai.FakeActivation{Version: v + 1, Network: "STAGING", Status: "ZONE_1", Submitted: now.Add(-100 * time.Minute)})
case 15, 16:
p.Latest, p.Staging = v+1, v+1
p.Activations = append(p.Activations, staged,
akamai.FakeActivation{Version: v + 1, Network: "PRODUCTION", Status: "PENDING", Submitted: now.Add(-time.Duration(i-10) * time.Minute)})
}
f.SetPMProperty(fmt.Sprintf("prp_9%05d", i), p)
if i == slowActivation {
slowBase = p
}
}
// Edge DNS.
records := 0
@@ -164,21 +163,57 @@ func fleetAkamai(f *akamai.Fake) string {
records += len(recs)
f.SetZone(fmt.Sprintf("zone%02d.example", z), akamai.FakeZone{Records: recs})
}
// Traffic: a long tail; CP code 400007 at 8% 5xx, 400019 at 30%.
// Traffic: a long tail, about one request in 2,000 failing.
for i := range 320 {
hits := int64(2_000_000 / (i + 1))
errs := hits / 2000
switch i {
case 7:
errs = hits * 8 / 100
case 19:
errs = hits * 30 / 100
}
f.SetTraffic(int64(400000+i), hits, errs)
f.SetTraffic(int64(400000+i), hits, hits/2000)
}
akamaiStory(f, now)
return fmt.Sprintf("%d GTM properties over %d datacenters, 180 CPS enrollments, 320 properties, 40 DNS zones with %d record sets, 320 CP codes", props, sites, records)
}
const (
legacyServer = "198.51.100.50" // the legacy property's one datacenter's server
legacyCPCode = 400019 // the legacy site's traffic
slowActivation = 15 // the property whose production activation drags
)
// slowBase is the slow property as fleetAkamai made it, before its new
// version.
var slowBase akamai.FakePMProperty
// akamaiStory puts the problems that come and go in their state now
// (scenario.go): story 3, the legacy datacenter; a load balancer that
// fails GTM's liveness test now and then; an activation that drags.
func akamaiStory(f *akamai.Fake, now time.Time) {
legacyDown := spellLegacyDC.active(now)
f.SetAlive("corp.example.akadns.net", "legacy", 3100, legacyServer, !legacyDown)
hits := int64(2_000_000 / (legacyCPCode - 400000 + 1))
errs := hits / 2000
if legacyDown {
errs = hits * 30 / 100
}
f.SetTraffic(legacyCPCode, hits, errs)
if sites := min(fleetSites, 6); sites > 2 {
f.SetAlive("shop.example.akadns.net", "api", 3100+2, fmt.Sprintf("203.0.%d.%d", 2, 11), !spellLBFlaps.active(now))
}
// Its new version goes to production at the start of each spell and
// is active an hour and twenty minutes later.
p := slowBase
p.Activations = append([]akamai.FakeActivation{}, p.Activations...)
v := p.Latest + 1
b := spellActivation.began(now)
p.Latest, p.Staging = v, v
p.Activations = append(p.Activations, akamai.FakeActivation{Version: v, Network: "STAGING", Status: "ACTIVE", Submitted: b.Add(-2 * time.Hour)})
if spellActivation.active(now) {
p.Activations = append(p.Activations, akamai.FakeActivation{Version: v, Network: "PRODUCTION", Status: "PENDING", Submitted: b})
} else {
p.Production = v
p.Activations = append(p.Activations, akamai.FakeActivation{Version: v, Network: "PRODUCTION", Status: "ACTIVE", Submitted: b})
}
f.SetPMProperty(fmt.Sprintf("prp_9%05d", slowActivation), p)
}
// fleetZone is zone z's ordinary records, with the given MX.
func fleetZone(z int, mx string) []akamai.RecordSet {
name := fmt.Sprintf("zone%02d.example", z)
+2 -9
View File
@@ -19,8 +19,7 @@ import (
// last one though its file was renewed, and site 5's sends it
// without its intermediate
// 30-39 PostgreSQL: each its own certificate from the internal CA
// (STARTTLS), and in /etc/pki/tls/certs; a few end soon, one
// has a 1024-bit key
// (STARTTLS), and in /etc/pki/tls/certs; one has a 1024-bit key
//
// Clusters have the router's (or ingress controller's) wildcard on every
// Route or Ingress, and cert-manager Certificates for ten of their apps.
@@ -76,12 +75,6 @@ func hostCerts(i int, name string) (*report.CertScan, *report.CertFiles) {
file("/etc/haproxy/certs/example.com.pem", c, wildcard.NotBefore.Add(time.Hour))
case pos >= 30 && pos < 40:
begun := time.Duration(30+(n*37)%300) * 24 * time.Hour
switch {
case n%97 == 30: // ends in a few days
begun = 360 * 24 * time.Hour
case n%131 == 31: // ended
begun = 370 * 24 * time.Hour
}
key := "ECDSA P-256"
if n%211 == 32 {
key = "RSA 1024"
@@ -158,7 +151,7 @@ func clusterCerts(name string, ocp bool, routes int, put func(report.Workload))
for n := range min(10, routes) {
app := appNames[n]
host := app + ".example.com"
begun := time.Duration(10+(n*7+k*3)%60) * 24 * time.Hour
begun := time.Duration(10+(n*7+k*3)%55) * 24 * time.Hour // all a month or more from their end, renewing at 30 days
if n == 4 && k == 0 {
begun = 86 * 24 * time.Hour // ends in four days: the renewal is failing
}
+8 -17
View File
@@ -17,11 +17,10 @@ import (
// dozen zones, a load-balanced pool per site whose origins are the sites'
// load balancers, and the last 15 minutes of traffic. obs-beacond reads it
// through cloudflare's stand_in (http on loopback only). Problems on
// purpose: legacy.example still uses Flexible SSL, partners.example's edge
// certificate ends in 9 days, media.example's advanced certificate is
// stuck validating, fleet-0052 fails its pool's health check from eastern
// North America, and 7% of shop.example's requests fail, most on
// checkout.shop.example.
// purpose: legacy.example still uses Flexible SSL; now and then fleet-0052
// fails its pool's health check from eastern North America; and while
// the shop's database replica is behind (scenario.go, story 2) 7% of
// shop.example's requests fail, most on checkout.shop.example.
const cloudflarePort = 58430
// cfToken is the fake's API token; not a secret, as the fake is loopback only.
@@ -109,16 +108,8 @@ func cfHandle(w http.ResponseWriter, r *http.Request) {
return
}
ends := now.Add(time.Duration(40+k*3) * 24 * time.Hour)
if z == "partners.example" {
ends = now.Add(9*24*time.Hour + 5*time.Hour)
}
packs := []map[string]any{{"type": "universal", "status": "active", "hosts": []string{z, "*." + z},
"certificates": []map[string]string{{"expires_on": ends.Format(time.RFC3339)}}}}
if z == "media.example" {
packs = append(packs, map[string]any{"type": "advanced", "status": "pending_validation", "hosts": []string{"cdn." + z},
"certificates": []map[string]string{}})
}
ok(packs)
ok([]map[string]any{{"type": "universal", "status": "active", "hosts": []string{z, "*." + z},
"certificates": []map[string]string{{"expires_on": ends.Format(time.RFC3339)}}}})
case len(parts) == 3 && parts[0] == "zones" && parts[2] == "load_balancers":
_, z, found := zone(parts[1])
if !found {
@@ -169,7 +160,7 @@ func cfHandle(w http.ResponseWriter, r *http.Request) {
for pos := 1; pos <= 2; pos++ {
_, a := cfOrigin(s, pos)
h := map[string]any{"healthy": true, "rtt": fmt.Sprintf("%dms", 20+ri*30+pos*3), "failure_reason": "No failures", "response_code": 200}
if s == 1 && pos == 2 && region == "ENAM" {
if s == 1 && pos == 2 && region == "ENAM" && spellLBFlaps.active(now) {
h = map[string]any{"healthy": false, "rtt": "0ms", "failure_reason": "Response code mismatch error", "response_code": 503}
}
origins = append(origins, map[string]any{a: h})
@@ -197,7 +188,7 @@ func cfHandle(w http.ResponseWriter, r *http.Request) {
return map[string]any{"count": n, "dimensions": map[string]any{"edgeResponseStatus": status, "securityAction": action, "clientRequestHTTPHost": host}}
}
win := []map[string]any{g(base, 200, "unknown", z), g(base/20, 304, "unknown", "www."+z), g(3, 502, "unknown", z), g(base/400, 403, "block", z)}
if z == "shop.example" {
if z == "shop.example" && spellReplicaBehind.active(now) {
win = append(win, g(base*6/100, 503, "unknown", "checkout.shop.example"), g(base/100, 502, "unknown", z))
}
zs = append(zs, map[string]any{"zoneTag": id, "win": win, "day": []map[string]int{{"count": base / 400 * 96}}})
+111 -6
View File
@@ -16,6 +16,7 @@ import (
"math"
mrand "math/rand/v2"
"net/http"
"reflect"
"slices"
"strconv"
"strings"
@@ -47,6 +48,105 @@ type fakeCluster struct {
noUsage bool // its reader sends none (Nutanix: usage is a separate API, not read yet)
source string // what a host agent calls it (a vCenter's host name); "" for a cluster agent
usageFn func(now time.Time) []report.Usage // its own readings instead of CPU and memory (an array's performance)
episodes []episode // records that go wrong and come right (scenario.go)
}
// episode is a record that takes its bad form while its spell is on.
type episode struct {
id string
s spell
good report.Workload
bad func(w *report.Workload, since time.Time)
on, in bool // on: in its bad form; in: played at least once
}
// flap makes record id go wrong during s and come right after
// (scenario.go): bad changes a copy of the record as it is now, given
// when the spell began.
func (c *fakeCluster) flap(id string, s spell, bad func(w *report.Workload, since time.Time)) {
if _, ok := c.recs[id]; !ok {
panic("flap: no record " + id)
}
c.episodes = append(c.episodes, episode{id: id, s: s, good: cloneWorkload(c.recs[id]), bad: bad})
c.play(time.Now())
}
// flapSet flaps every record that bad has and good hasn't, or has
// otherwise: a reader's records made from a good and a bad snapshot. A
// record only bad has is there during the spell and gone after.
func (c *fakeCluster) flapSet(s spell, good, bad []report.Workload) {
have := map[string]report.Workload{}
for _, r := range good {
have[r.ID] = r
}
for _, b := range bad {
g, ok := have[b.ID]
if ok {
g.Since, b.Since = time.Time{}, time.Time{}
if reflect.DeepEqual(g, b) {
continue
}
}
b := cloneWorkload(b)
if !ok {
c.episodes = append(c.episodes, episode{id: b.ID, s: s, bad: func(w *report.Workload, since time.Time) { *w = cloneWorkload(b); w.Since = since }})
continue
}
c.flap(b.ID, s, func(w *report.Workload, since time.Time) { *w = cloneWorkload(b); w.Since = since })
}
c.play(time.Now())
}
// play puts each flapping record in the form its spell says, and marks
// the ones that changed.
func (c *fakeCluster) play(now time.Time) {
c.mu.Lock()
defer c.mu.Unlock()
for i := range c.episodes {
e := &c.episodes[i]
on := e.s.active(now)
if e.in && on == e.on {
continue
}
e.on, e.in = on, true
if e.good.ID == "" && !on { // there only during the spell
if _, ok := c.recs[e.id]; ok {
delete(c.recs, e.id)
delete(c.changed, e.id)
c.removed = append(c.removed, e.id)
}
continue
}
w := cloneWorkload(e.good)
if on {
e.bad(&w, e.s.began(now))
} else if !e.good.Since.IsZero() && now.Sub(e.s.began(now)) < e.s.every {
w.Since = e.s.began(now).Add(e.s.on) // right again since the spell ended
}
c.recs[e.id] = w
c.changed[e.id] = true
}
}
// flapping says whether record id has an episode, so churn leaves it be.
func (c *fakeCluster) flapping(id string) bool {
for _, e := range c.episodes {
if e.id == id {
return true
}
}
return false
}
func cloneWorkload(w report.Workload) report.Workload {
if w.Facts != nil {
f := make(map[string]any, len(w.Facts))
for k, v := range w.Facts {
f[k] = v
}
w.Facts = f
}
return w
}
func newID() string {
@@ -69,6 +169,7 @@ func newCluster(name, token string, s clusterShape) *fakeCluster {
Since: now.Add(-48 * time.Hour), Facts: map[string]any{"kubelet": kubelet, "os": nodeOS, "cpus": 32, "memory_gib": 256}})
}
k := clusterIndex(name)
crasher := ""
for i := range s.controllers {
ns, owner := fmt.Sprintf("ns-%02d", i%30), fmt.Sprintf("deployment/app-%03d", i)
want := s.pods / max(s.controllers, 1)
@@ -82,11 +183,8 @@ func newCluster(name, token string, s clusterShape) *fakeCluster {
for range want {
c.addPod(ns, owner, now)
}
if i%97 == 0 { // a few controllers with a crashing pod
id := c.pods[len(c.pods)-1]
p := c.recs[id]
p.State, p.Word, p.Facts = report.StateCrashloop, "Restarting (CrashLoopBackOff)", map[string]any{"restarts": 40, "last_exit": "Error"}
put(p)
if i == 97 && k == 0 { // a pod that crashes for a while after each deploy (scenario.go)
crasher = c.pods[len(c.pods)-1]
}
}
vmGroup := map[string][]string{}
@@ -142,6 +240,12 @@ func newCluster(name, token string, s clusterShape) *fakeCluster {
if name == "cluster-00" { // the pod the probe changes
put(report.Workload{Kind: report.KindPod, ID: "probe", Name: "probe", Namespace: "fleet", Node: "node-00", State: report.StateOK, Word: "Running"})
}
if crasher != "" {
c.flap(crasher, spellCrashloop, func(p *report.Workload, since time.Time) {
p.State, p.Word, p.Since = report.StateCrashloop, "Restarting (CrashLoopBackOff)", since
p.Facts["restarts"], p.Facts["last_exit"], p.Facts["ready"] = 9, "Error", "0/1"
})
}
return c
}
@@ -212,7 +316,7 @@ func (c *fakeCluster) churn(n int, now time.Time) {
i := mrand.N(len(c.pods))
id := c.pods[i]
p := c.recs[id]
if p.State == report.StateCrashloop {
if p.State == report.StateCrashloop || c.flapping(id) {
continue
}
if mrand.N(10) == 0 { // replaced: a new pod, the old one gone
@@ -372,6 +476,7 @@ func runCluster(ctx context.Context, c *fakeCluster, client *http.Client, pick f
}
}
c.churn(int(c.shape.churn*5), now)
c.play(now)
if probed != nil {
probeWord, probeAt = c.probe(now), now
}
+8 -3
View File
@@ -20,7 +20,9 @@ import (
// real network; the status pages come in the hosts' reports, as an agent's
// fetches do.
// Problems on purpose: the leader is fleet-0031, not its usual fleet-0030
// (a failover nobody moved back), and fleet-0032 is 40 MB behind.
// (a failover nobody moved back), and every so often fleet-0032 falls 40
// MB behind and HAProxy takes it out of the replicas' pool until it
// catches up (scenario.go, story 2).
const patroniPort = 58420
var (
@@ -85,7 +87,9 @@ func servePatroni(ctx context.Context) {
case pgHome:
role = "sync_standby"
case pgLagging:
lag = `,"lag":41943040`
if spellReplicaBehind.active(time.Now()) {
lag = `,"lag":41943040`
}
}
ms = append(ms, fmt.Sprintf(`{"name":%q,"role":%q,"state":%q,"host":%q,"port":5432,"timeline":7%s}`, hostName(m), role, state, pgAddr(m), lag))
}
@@ -131,7 +135,8 @@ func dataFetches(n int, now time.Time) []report.Fetch {
fmt.Fprintf(&b, "%s,BACKEND,0,UP,1\n", name)
}
backend("pg-primary", func(m int) bool { return m == pgLeader }, pgMembers)
backend("pg-replicas", func(m int) bool { return m != pgLeader }, pgMembers)
behind := spellReplicaBehind.active(now)
backend("pg-replicas", func(m int) bool { return m != pgLeader && !(behind && m == pgLagging) }, pgMembers)
var apps []int
for p := 3; p < 13; p++ {
apps = append(apps, p)
+72 -21
View File
@@ -16,17 +16,20 @@ import (
// keep running, and the backups each site keeps, so that every page has
// something to show. config writes the files obs-beacond reads; run makes
// the hosts report against them. Both work them out from the number of
// agents, so give both the same -n. Problems on purpose:
// agents, so give both the same -n. Problems on purpose, most of them
// coming and going (scenario.go):
//
// checks legacy.example.com answers 502 (a warning-only check);
// billing.example.com's certificate ends in 9 days; the
// ping to the last site loses 3%
// units fleet-0041's redis-server has failed
// containers fleet-0007's app-04 exited; fleet-0008's app-09 is stopped
// on purpose
// backups site 1's newest pg-dump is 40 hours old; site 2's shrank to
// a third; site 3's backup disk is 93% full; site 4's
// pg-backup job failed; site 5's md1 is missing a disk
// checks legacy.example.com answers 502 now and then (a warning-only
// check); billing.example.com's certificate ends in 9 days;
// the ping to the last site loses 3%; mx2 lands on a block
// list now and then
// units fleet-0041's redis-server fails now and then, and is
// restarted
// containers fleet-0007's app-04 exits now and then; fleet-0008's app-09
// is stopped on purpose
// backups site 1's pg-dump is late now and then; site 2's shrinks to a
// third now and then; site 3's backup disk is 93% full; site
// 4's pg-backup job failed; site 5's md1 is missing a disk
// role is what a host does in its site, by position (netmap.go).
func role(pos int) string {
@@ -112,6 +115,9 @@ func demoChecks(hosts int) []demoCheck {
out = append(out,
demoCheck{id: "web-legacy", name: "legacy.example.com", yaml: "http: https://legacy.example.com/\n severity: warning", from: web,
result: func(from int, now time.Time) report.CheckResult {
if !spellLegacy502.active(now) {
return report.CheckResult{ID: "web-legacy", At: now.UTC(), OK: true, Status: 200, Ms: 340}
}
return report.CheckResult{ID: "web-legacy", At: now.UTC(), Status: 502, Ms: 2100, Error: "HTTP 502, want 200"}
}},
demoCheck{id: "dns-mx", name: "example.com MX", yaml: "dns: example.com\n type: MX\n contains: mx1.example.com", from: web,
@@ -126,6 +132,9 @@ func demoChecks(hosts int) []demoCheck {
// mx2 landed on a list that flags whole networks: a warning to look at, not a page.
demoCheck{id: "rbl-mx2", name: "mx2.example.com on block lists", yaml: "rbl: 192.0.2.26\n severity: warning", from: web[:1],
result: func(from int, now time.Time) report.CheckResult {
if !spellBlocklisted.active(now) {
return report.CheckResult{ID: "rbl-mx2", At: now.UTC(), OK: true, Ms: 190, Answer: "clean on 5 of 5"}
}
return report.CheckResult{ID: "rbl-mx2", At: now.UTC(), Ms: 210, Answer: "clean on 4 of 5",
Error: "listed on dnsbl-1.uceprotect.net (127.0.0.2)"}
}},
@@ -204,16 +213,19 @@ func demoScan(n, hosts int, now time.Time) *report.Backups {
if name == "system" {
size = int64(800+40*k) << 20
}
d := report.BackupDir{Name: name, From: hostName(from), Path: fmt.Sprintf("/srv/backups/%s/%s/%s", hostName(from), name, now.Format("2006-01-02")),
Size: size, PrevSize: size - size/50, Count: 14, Modified: now.Add(-time.Duration(3+k) * time.Hour).UTC()}
d := report.BackupDir{Name: name, From: hostName(from), Size: size, PrevSize: size - size/50, Count: 14,
Modified: now.Add(-time.Duration(3+k) * time.Hour).UTC()}
if k == 0 && name == "pg-dump" {
switch site {
case 1:
d.Modified = now.Add(-40 * time.Hour).UTC()
case 2:
switch {
case site == 1 && spellLateBackup.active(now):
// Last night's never came: the newest is the night before's.
d.Modified = spellLateBackup.began(now).Add(-26*time.Hour - 10*time.Minute).UTC()
case site == 2 && spellBackupShrank.active(now):
d.PrevSize, d.Size = d.Size, d.Size/3
}
}
// Each copy is in a folder named after the day it was made.
d.Path = fmt.Sprintf("/srv/backups/%s/%s/%s", hostName(from), name, d.Modified.Format("2006-01-02"))
sc.Received = append(sc.Received, d)
}
}
@@ -328,22 +340,60 @@ func (d *demoHost) setup(r *report.Report) {
}
for k := range r.Containers.List {
c := &r.Containers.List[k]
switch {
case d.n == demoExited && c.Name == "app-04":
c.State, c.Status, c.Health = "exited", "Exited (137) 12 minutes ago", ""
case d.n == demoStoppedOnPurp && c.Name == "app-09":
if d.n == demoStoppedOnPurp && c.Name == "app-09" {
c.State, c.Status, c.Health = "exited", "Exited (0) 2 days ago", ""
}
}
}
// containers puts the containers that come and go in their state now.
func (d *demoHost) containers(r *report.Report, now time.Time) {
if d.n != demoExited || r.Containers == nil {
return
}
list := append([]report.Container{}, r.Containers.List...)
for k := range list {
c := &list[k]
if c.Name != "app-04" {
continue
}
if b := spellAppExits.began(now); spellAppExits.active(now) {
c.State, c.Status, c.Health = "exited", fmt.Sprintf("Exited (137) %s ago", fuzzy(now.Sub(b))), ""
} else {
c.State, c.Health = "running", "healthy"
c.Status = fmt.Sprintf("Up %s (healthy)", fuzzy(now.Sub(b.Add(spellAppExits.on))))
}
}
r.Containers = &report.Containers{List: list}
}
// fuzzy is a duration as docker ps says it.
func fuzzy(d time.Duration) string {
switch {
case d < time.Minute:
return "Less than a minute"
case d < 2*time.Minute:
return "About a minute"
case d < time.Hour:
return fmt.Sprintf("%d minutes", int(d.Minutes()))
case d < 2*time.Hour:
return "About an hour"
}
return fmt.Sprintf("%d hours", int(d.Hours()))
}
// stamp adds the units, check results and backup scan to a report.
func (d *demoHost) stamp(r *report.Report, start, now time.Time) {
r.Units = nil
for _, u := range demoUnits(d.n) {
un := report.Unit{Name: u + ".service", Load: "loaded", Active: "active", Sub: "running", Since: start.UTC()}
if d.n == demoFailedUnit && u == "redis-server" {
un.Active, un.Sub, un.Restarts, un.Since = "failed", "failed", 5, time.Time{}
switch b := spellRedisFails.began(now); {
case spellRedisFails.active(now):
un.Active, un.Sub, un.Restarts, un.Since = "failed", "failed", 5, time.Time{}
case b.After(start): // restarted by hand when the spell ended
un.Since = b.Add(spellRedisFails.on).UTC()
}
}
r.Units = append(r.Units, un)
}
@@ -351,6 +401,7 @@ func (d *demoHost) stamp(r *report.Report, start, now time.Time) {
for _, c := range d.checks {
r.Checks = append(r.Checks, c.result(d.n, now))
}
d.containers(r, now)
r.Backups = demoScan(d.n, d.hosts, now)
if hasDataLayer(d.hosts) {
r.Fetches = dataFetches(d.n, now)
+4 -9
View File
@@ -2,7 +2,6 @@ package main
import (
"fmt"
"strings"
"time"
"github.com/cairnobs/cairnobs/internal/hmc"
@@ -18,9 +17,9 @@ const hmcName = "hmc1"
// newHMC is three servers: p10-prod runs the four IBM i partitions (their
// serial and partition numbers are the ones the partitions say), an AIX
// database and a Linux pair; p10-dr has its attention light on, a VIOS
// the HMC can't reach over RMC and a Linux partition stopped in Open
// Firmware; p9-old is in standby, waiting to be retired.
// database and a Linux pair; p10-dr has its attention light on (a
// serviceable event nobody has closed); p9-old is in standby, waiting to
// be retired.
func newHMC(name, token string) *fakeCluster {
c := &fakeCluster{name: name, token: token, recs: map[string]report.Workload{}, changed: map[string]bool{}, noUsage: true, source: hmcName}
n := 0
@@ -52,9 +51,8 @@ func newHMC(name, token string) *fakeCluster {
MachineType: "9105", Model: "22B", Serial: "78DEF45", ProcUnits: 20, ProcFree: 12.25, MemMB: 1 << 20, MemFreeMB: 655_360, AttentionLED: "true",
Partitions: []hmc.Partition{vios("p10-dr", 1, 1), vios("p10-dr", 2, 2),
part("p10-dr", "aixdb1-dr", "AIX/Linux", "AIX 7.3 7300-03-00-2446", "running", 3, 2, 8, 512),
part("p10-dr", "lnxtest1", "AIX/Linux", "Linux/SUSE 6.4.0-150600.23.25-default 15.6", "open firmware", 4, 0.5, 2, 16),
part("p10-dr", "lnxtest1", "AIX/Linux", "Linux/SUSE 6.4.0-150600.23.25-default 15.6", "running", 4, 0.5, 2, 16),
part("p10-dr", "aixtest1", "AIX/Linux", "Unknown", "not activated", 5, 0.5, 1, 16)}}
dr.Partitions[1].RMC = "inactive"
old := hmc.Server{UUID: "1f7a9c3e-5b2d-3e84-a6c0-94d2e8b1f733", Name: "p9-old", State: "standby", DetailedState: "None",
MachineType: "9009", Model: "42A", Serial: "13XYZ99", ProcUnits: 16, ProcFree: 16, MemMB: 512 << 10, MemFreeMB: 512 << 10, AttentionLED: "false",
Partitions: []hmc.Partition{part("p9-old", "aixold1", "AIX/Linux", "AIX 7.2 7200-05-08-2420", "not activated", 1, 2, 4, 64)}}
@@ -62,9 +60,6 @@ func newHMC(name, token string) *fakeCluster {
now := time.Now()
for i, r := range hmc.Records(s) {
r.Since = now.Add(-time.Duration(30+i%11) * 24 * time.Hour)
if strings.HasPrefix(r.Word, "Stopped in Open Firmware") || strings.HasPrefix(r.Word, "Running, but the HMC") {
r.Since = now.Add(-time.Duration(2+i%5) * time.Hour)
}
c.recs[r.ID] = r
}
return c
+57 -45
View File
@@ -20,12 +20,11 @@ const ibmiHost = 41
// ibmiPartitions are the partitions, in the order -ibmi N takes them.
var ibmiPartitions = []string{"prod1", "prod2", "ha1", "dev1"}
// newIBMi is the k'th partition: prod1 busy, its system ASP past IBM i's
// threshold, a disk unit's parity being rebuilt and the nightly batch
// stuck on a question; prod2 with ORDERS ended, its job queue filling,
// and its backup interface down; ha1 the PowerHA standby, its
// independent ASP varied off; dev1 on 7.4 without the authority to read
// its Technology Refresh.
// newIBMi is the k'th partition: prod1 busy, its nightly batch stuck on a
// question now and then; prod2's ORDERS subsystem ended for a while now
// and then, its job queue filling meanwhile (scenario.go); ha1 the
// PowerHA standby, its independent ASP varied off; dev1 on 7.4 without
// the authority to read its Technology Refresh.
func newIBMi(name, token string, k int) *fakeCluster {
part := ibmiPartitions[k]
c := &fakeCluster{name: name, token: token, recs: map[string]report.Workload{}, changed: map[string]bool{}, noUsage: true, source: part}
@@ -39,49 +38,62 @@ func newIBMi(name, token string, k int) *fakeCluster {
pct := min(100, base*(0.6+0.4*day)*(0.9+0.2*rand.Float64()))
return []report.Usage{{ID: report.ClusterSelf, CPU: math.Round(pct*units) / 100, CPUOf: units}}
}
sysUsed := []float64{91.4, 72.8, 64.1, 48.0}[k]
s := &ibmi.Snapshot{Name: part, Release: "7.5", TR: 6, At: time.Now(),
System: ibmi.System{HostName: "S" + part, PartitionName: strings.ToUpper(part), PartitionID: k + 2, Partitions: 6, MachineType: "9105", MachineModel: "42A",
Serial: "78ABC12", CPUs: int(units * 2), Capacity: units, MainStorageKB: int64(units*64) << 20, SystemASPMB: 2_400_000,
SystemASPUsed: sysUsed, TempMB: 38_000, Jobs: []int{18204, 14550, 6120, 3307}[k], MaxJobs: 163520,
ActiveJobs: []int{612, 488, 201, 97}[k], Threads: []int{2870, 2210, 890, 402}[k]},
ASPs: []ibmi.ASP{{Number: 1, State: "NONE", Type: "SYSTEM", Units: 8, Present: "ALL", TotalMB: 2_400_000,
FreeMB: int64(2_400_000 * (100 - sysUsed) / 100), Threshold: 90}}}
switch part {
case "prod1", "ha1":
ia := ibmi.ASP{Number: 33, Name: "PRODIASP", State: "AVAILABLE", Type: "PRIMARY", Units: 6, Present: "ALL", TotalMB: 3_600_000, FreeMB: 1_440_000, Threshold: 90}
if part == "ha1" {
ia.State = "VARIED OFF"
sysUsed := []float64{84.6, 72.8, 64.1, 48.0}[k]
snap := func(bad bool) *ibmi.Snapshot {
s := &ibmi.Snapshot{Name: part, Release: "7.5", TR: 6, At: time.Now(),
System: ibmi.System{HostName: "S" + part, PartitionName: strings.ToUpper(part), PartitionID: k + 2, Partitions: 6, MachineType: "9105", MachineModel: "42A",
Serial: "78ABC12", CPUs: int(units * 2), Capacity: units, MainStorageKB: int64(units*64) << 20, SystemASPMB: 2_400_000,
SystemASPUsed: sysUsed, TempMB: 38_000, Jobs: []int{18204, 14550, 6120, 3307}[k], MaxJobs: 163520,
ActiveJobs: []int{612, 488, 201, 97}[k], Threads: []int{2870, 2210, 890, 402}[k]},
ASPs: []ibmi.ASP{{Number: 1, State: "NONE", Type: "SYSTEM", Units: 8, Present: "ALL", TotalMB: 2_400_000,
FreeMB: int64(2_400_000 * (100 - sysUsed) / 100), Threshold: 90}}}
switch part {
case "prod1", "ha1":
ia := ibmi.ASP{Number: 33, Name: "PRODIASP", State: "AVAILABLE", Type: "PRIMARY", Units: 6, Present: "ALL", TotalMB: 3_600_000, FreeMB: 1_440_000, Threshold: 90}
if part == "ha1" {
ia.State = "VARIED OFF"
}
s.ASPs = append(s.ASPs, ia)
case "dev1":
s.Release, s.TR = "7.4", 0
}
s.ASPs = append(s.ASPs, ia)
case "dev1":
s.Release, s.TR = "7.4", 0
}
sbs := func(lib, name, status string, jobs int) ibmi.Subsystem {
return ibmi.Subsystem{Library: lib, Name: name, Status: status, Jobs: jobs, Controlling: name == "QCTL"}
}
s.Subsys = []ibmi.Subsystem{sbs("QSYS", "QCTL", "ACTIVE", 3), sbs("QSYS", "QINTER", "ACTIVE", 20+k*7), sbs("QSYS", "QBATCH", "ACTIVE", 6+k),
sbs("QSYS", "QUSRWRK", "ACTIVE", 60+k*9), sbs("QSYS", "QSERVER", "ACTIVE", 14), sbs("QSYS", "QSYSWRK", "ACTIVE", 110), sbs("QSYS", "QSPL", "ACTIVE", 4),
sbs("QSYS", "QHTTPSVR", "ACTIVE", 9), sbs("QSYS", "QSNADS", "INACTIVE", 0), sbs("QSYS", "QCMN", "ACTIVE", 2), sbs("ORDLIB", "ORDERS", "ACTIVE", 12)}
s.Ifaces = []ibmi.Interface{{Address: fmt.Sprintf("10.0.4.%d", 20+k), Prefix: 24, Line: "ETHLINE", Status: "ACTIVE", Autostart: true},
{Address: fmt.Sprintf("10.0.9.%d", 20+k), Prefix: 24, Line: "ETHBKP", Status: "ACTIVE", Autostart: true}}
switch part {
case "prod1":
s.Disks = []ibmi.Disk{{ASP: 1, Unit: 6, Resource: "DMP006", Hardware: "PARITY REBUILD"}}
s.Inquiries = []ibmi.Message{{Key: "0001A2F0", ID: "CPA0701", Inquiry: true, Severity: 99, Job: "418276/BATCH/NIGHTLY", User: "BATCH",
Text: "MCH1211 received by procedure ORDPOST. (C D I R)", Age: 47 * time.Minute}}
s.Unanswered = 1
case "prod2":
s.Subsys[len(s.Subsys)-1].Status, s.Subsys[len(s.Subsys)-1].Jobs = "INACTIVE", 0 // ORDERS: ended, not started again
s.Ifaces[1].Status = "INACTIVE" // its backup line, set to start
s.JobQueues = []ibmi.JobQueue{{Library: "ORDLIB", Name: "ORDERSQ", Waiting: 14}} // ORDERS isn't there to take them
case "ha1":
s.Subsys = s.Subsys[:len(s.Subsys)-1] // ORDERS runs on the active node only
sbs := func(lib, name, status string, jobs int) ibmi.Subsystem {
return ibmi.Subsystem{Library: lib, Name: name, Status: status, Jobs: jobs, Controlling: name == "QCTL"}
}
s.Subsys = []ibmi.Subsystem{sbs("QSYS", "QCTL", "ACTIVE", 3), sbs("QSYS", "QINTER", "ACTIVE", 20+k*7), sbs("QSYS", "QBATCH", "ACTIVE", 6+k),
sbs("QSYS", "QUSRWRK", "ACTIVE", 60+k*9), sbs("QSYS", "QSERVER", "ACTIVE", 14), sbs("QSYS", "QSYSWRK", "ACTIVE", 110), sbs("QSYS", "QSPL", "ACTIVE", 4),
sbs("QSYS", "QHTTPSVR", "ACTIVE", 9), sbs("QSYS", "QSNADS", "INACTIVE", 0), sbs("QSYS", "QCMN", "ACTIVE", 2), sbs("ORDLIB", "ORDERS", "ACTIVE", 12)}
s.Ifaces = []ibmi.Interface{{Address: fmt.Sprintf("10.0.4.%d", 20+k), Prefix: 24, Line: "ETHLINE", Status: "ACTIVE", Autostart: true},
{Address: fmt.Sprintf("10.0.9.%d", 20+k), Prefix: 24, Line: "ETHBKP", Status: "ACTIVE", Autostart: true}}
switch {
case part == "prod1" && bad:
s.Inquiries = []ibmi.Message{{Key: "0001A2F0", ID: "CPA0701", Inquiry: true, Severity: 99, Job: "418276/BATCH/NIGHTLY", User: "BATCH",
Text: "MCH1211 received by procedure ORDPOST. (C D I R)", Age: 7 * time.Minute}}
s.Unanswered = 1
case part == "prod2" && bad:
s.Subsys[len(s.Subsys)-1].Status, s.Subsys[len(s.Subsys)-1].Jobs = "INACTIVE", 0 // ORDERS: ended, not started again
s.JobQueues = []ibmi.JobQueue{{Library: "ORDLIB", Name: "ORDERSQ", Waiting: 14}} // ORDERS isn't there to take them
case part == "ha1":
s.Subsys = s.Subsys[:len(s.Subsys)-1] // ORDERS runs on the active node only
}
return s
}
now := time.Now()
for _, r := range ibmi.Records(s) {
r.Since = now.Add(-time.Duration(40+k*7) * 24 * time.Hour)
recs := func(bad bool) []report.Workload {
out := ibmi.Records(snap(bad))
for i := range out {
out[i].Since = now.Add(-time.Duration(40+k*7) * 24 * time.Hour)
}
return out
}
for _, r := range recs(false) {
c.recs[r.ID] = r
}
switch part {
case "prod1":
c.flapSet(spellInquiry, recs(false), recs(true))
case "prod2":
c.flapSet(spellOrdersDown, recs(false), recs(true))
}
return c
}
+3 -6
View File
@@ -589,8 +589,8 @@ const (
var vmRoles = [libvirtVMs]string{"db", "web", "cache", "build", "mq", "ldap", "backup", "monitor", "runner", "jump", "proxy", "spare"}
// hypervisorVMs are host i's domains: all running, but its spare is shut
// off, and on the third hypervisor one has crashed.
// hypervisorVMs are host i's domains: all running but its spare, which is
// shut off.
func hypervisorVMs(i int, now time.Time) []report.Workload {
out := make([]report.Workload, libvirtVMs)
for k := range out {
@@ -598,11 +598,8 @@ func hypervisorVMs(i int, now time.Time) []report.Workload {
r := report.Workload{Kind: report.KindVM, ID: id, Name: fmt.Sprintf("%s-%02d", vmRoles[k], i/hypervisorEvery),
State: report.StateOK, Word: "Running", Since: now.Add(-time.Duration(3+k) * 24 * time.Hour).Truncate(time.Second),
Facts: map[string]any{"firmware_uuid": id, "autostart": k != libvirtVMs-1, "hypervisor": "libvirt", "vcpus": 2 << (k % 3), "memory_gib": float64(int(4) << (k % 3))}}
switch {
case k == libvirtVMs-1:
if k == libvirtVMs-1 {
r.State, r.Word = report.StateStopped, "Shut off"
case i/hypervisorEvery == 2 && k == 7:
r.State, r.Word, r.Since = report.StateDown, "Crashed", now.Add(-47*time.Minute).Truncate(time.Second)
}
out[k] = r
}
+3 -9
View File
@@ -35,21 +35,12 @@ func newNutanix(name, token string, vms int) *fakeCluster {
r := report.Workload{Kind: report.KindNode, ID: fmt.Sprintf("a1b2c3d4-%04d-4000-8000-%012d", cl, n), Name: host(cl, n),
State: report.StateOK, Word: "Connected", Since: now.Add(-time.Duration(20+n) * 24 * time.Hour),
Facts: map[string]any{"cluster": peName(cl), "cpus": 64, "memory_gib": 1024.0, "version": "AHV 10.0.1", "model": "NX-8170-G9"}}
switch {
case cl == 1 && n == 4:
r.State, r.Word, r.Since = report.StateUpdating, "In maintenance mode", now.Add(-20*time.Minute)
case cl == 2 && n == 2:
r.State, r.Word, r.Facts["alerts"] = report.StateDegraded, "Alert from Prism: Disk Failure", "Disk Failure (critical)"
}
put(r)
}
for k, ct := range []string{"default-container", "vdi-images", "db-data"} {
r := report.Workload{Kind: report.KindPVC, ID: fmt.Sprintf("c0ffee00-%04d-4000-8000-%012d", cl, k), Name: ct,
State: report.StateOK, Word: "Available", Since: now.Add(-200 * 24 * time.Hour),
Facts: map[string]any{"type": "storage container", "cluster": peName(cl), "capacity_gib": 81920.0, "replication_factor": 2}}
if cl == 0 && k == 2 {
r.State, r.Word, r.Facts["alerts"] = report.StateDegraded, "Alert from Prism: Storage Container Space Usage Exceeded", "Storage Container Space Usage Exceeded (critical)"
}
put(r)
}
}
@@ -72,5 +63,8 @@ func newNutanix(name, token string, vms int) *fakeCluster {
}
put(r)
}
c.flap(fmt.Sprintf("a1b2c3d4-%04d-4000-8000-%012d", 1, 4), spellAHVUpdate, func(r *report.Workload, since time.Time) { // an AOS upgrade's rolling restart (scenario.go)
r.State, r.Word, r.Since = report.StateUpdating, "In maintenance mode", since
})
return c
}
+3 -3
View File
@@ -45,9 +45,6 @@ func newProxmox(name, token string, guests int) *fakeCluster {
switch {
case k%23 == 22:
r.State, r.Word, r.Facts["ha"] = report.StateStopped, "Stopped", "stopped"
case k == 41:
r.State, r.Word, r.Facts["ha"] = report.StateDown, "HA can't run it (error)", "error"
r.Since = now.Add(-18 * time.Minute)
case k == 57:
r.State, r.Word, r.Facts["lock"] = report.StateUpdating, "Migrating", "migrate"
case k%50 == 9:
@@ -55,5 +52,8 @@ func newProxmox(name, token string, guests int) *fakeCluster {
}
put(r)
}
c.flap("lxc/141", spellPVEHA, func(r *report.Workload, since time.Time) { // scenario.go
r.State, r.Word, r.Since, r.Facts["ha"] = report.StateDown, "HA can't run it (error)", since, "error"
})
return c
}
+27 -23
View File
@@ -27,25 +27,21 @@ func newPure(name, token string, k int) *fakeCluster {
}
// newFlashBlade is fb-prod-01 (docs/pure.md, step 5): ten blades, one
// evacuating; a failed fan; 400 file systems, cluster-02's first 100 VM
// disks among them (Portworx names them after their PersistentVolumes),
// one whose replication to fb-dr-01 is unhealthy; 40 buckets.
// evacuating now and then (scenario.go); 400 file systems, cluster-02's
// first 100 VM disks among them (Portworx names them after their
// PersistentVolumes); 40 buckets.
func newFlashBlade(name, token string, k int) *fakeCluster {
arr := pureArrays[k]
c := &fakeCluster{name: name, token: token, recs: map[string]report.Workload{}, changed: map[string]bool{}, noUsage: true, source: arr}
c.usageFn = func(now time.Time) []report.Usage { return pureIO(k, now) }
now := time.Now()
put := func(w report.Workload) { c.recs[w.ID] = w }
put(report.Workload{Kind: report.KindCluster, ID: report.ClusterSelf, Name: arr, State: report.StateDegraded, Word: "Hardware: CH1.FAN2 critical (Fan speed is 0 RPM)",
Since: now.Add(-95 * time.Minute), Facts: map[string]any{"distribution": "Purity//FB", "version": "4.5.6", "model": "FlashBlade//S", "capacity_tib": 2000.0,
"used_pct": 61.0, "data_reduction": 3.1, "hardware": "CH1.FAN2 critical (Fan speed is 0 RPM)", "alerts": "CH1.FAN2: Fan failure (critical)"}})
put(report.Workload{Kind: report.KindCluster, ID: report.ClusterSelf, Name: arr, State: report.StateOK, Word: "API answering",
Since: now.Add(-200 * 24 * time.Hour), Facts: map[string]any{"distribution": "Purity//FB", "version": "4.5.6", "model": "FlashBlade//S", "capacity_tib": 2000.0,
"used_pct": 61.0, "data_reduction": 3.1}})
for i := 1; i <= 10; i++ {
r := report.Workload{Kind: report.KindNode, ID: fmt.Sprintf("ch1.fb%d", i), Name: fmt.Sprintf("CH1.FB%d", i), State: report.StateOK, Word: "Healthy",
Since: now.Add(-200 * 24 * time.Hour), Facts: map[string]any{"raw_tib": 48.0}}
if i == 6 {
r.State, r.Word, r.Since = report.StateUpdating, "Evacuating its data (37%)", now.Add(-2*time.Hour)
}
put(r)
put(report.Workload{Kind: report.KindNode, ID: fmt.Sprintf("ch1.fb%d", i), Name: fmt.Sprintf("CH1.FB%d", i), State: report.StateOK, Word: "Healthy",
Since: now.Add(-200 * 24 * time.Hour), Facts: map[string]any{"raw_tib": 48.0}})
}
put(report.Workload{Kind: report.KindCluster, ID: "connection/c-fb-dr-01", Name: "connection/fb-dr-01", State: report.StateOK, Word: "Connected",
Since: now.Add(-90 * 24 * time.Hour), Facts: map[string]any{"type": "async-replication", "version": "4.5.6"}})
@@ -62,9 +58,7 @@ func newFlashBlade(name, token string, k int) *fakeCluster {
r.Name, r.Word, r.Facts["protocols"] = "home", "Shared over NFSv3, NFSv4.1, SMB", "NFSv3, NFSv4.1, SMB"
r.Facts["replication"] = "to fb-dr-01, replicating, 4 min behind"
case n == 101:
r.Name, r.State = "scratch", report.StateDegraded
r.Word = "Replication to fb-dr-01 is unhealthy: Target file system is full"
r.Facts["replication"] = "to fb-dr-01, unhealthy, 5 h behind"
r.Name = "scratch"
case n%50 == 7:
r.Word = "Not shared over any protocol"
delete(r.Facts, "protocols")
@@ -80,6 +74,10 @@ func newFlashBlade(name, token string, k int) *fakeCluster {
}
put(r)
}
c.flap("ch1.fb6", spellBladeEvac, func(r *report.Workload, since time.Time) {
done := min(95, 5+int(time.Since(since).Minutes()))
r.State, r.Word, r.Since = report.StateUpdating, fmt.Sprintf("Evacuating its data (%d%%)", done), since
})
return c
}
@@ -99,12 +97,8 @@ func newFlashArray(name, token string, k, volumes int) *fakeCluster {
}
put(self)
for n, mode := range []string{"primary", "secondary"} {
r := report.Workload{Kind: report.KindNode, ID: fmt.Sprintf("ct%d", n), Name: fmt.Sprintf("CT%d", n), State: report.StateOK, Word: "Ready, " + mode,
Since: now.Add(-40 * 24 * time.Hour), Facts: map[string]any{"mode": mode, "model": "FA-X70R4", "version": "6.7.3"}}
if arr == "fa-vdi-01" && n == 1 {
r.State, r.Word, r.Since = report.StateUpdating, "Updating", now.Add(-12*time.Minute)
}
put(r)
put(report.Workload{Kind: report.KindNode, ID: fmt.Sprintf("ct%d", n), Name: fmt.Sprintf("CT%d", n), State: report.StateOK, Word: "Ready, " + mode,
Since: now.Add(-40 * 24 * time.Hour), Facts: map[string]any{"mode": mode, "model": "FA-X70R4", "version": "6.7.3"}})
}
switch arr {
case "fa-prod-01", "fa-prod-02":
@@ -116,8 +110,8 @@ func newFlashArray(name, token string, k, volumes int) *fakeCluster {
put(report.Workload{Kind: report.KindCluster, ID: "connection/c-fa-dr-01", Name: "connection/fa-dr-01", State: report.StateOK, Word: "Connected",
Since: now.Add(-60 * 24 * time.Hour), Facts: map[string]any{"type": "async-replication", "version": "6.7.3"}})
case "fa-dr-01":
put(report.Workload{Kind: report.KindCluster, ID: "connection/c-fa-prod-01", Name: "connection/fa-prod-01", State: report.StateDegraded,
Word: "Connected, but its paths aren't redundant", Since: now.Add(-50 * time.Minute), Facts: map[string]any{"type": "async-replication", "version": "6.7.3"}})
put(report.Workload{Kind: report.KindCluster, ID: "connection/c-fa-prod-01", Name: "connection/fa-prod-01", State: report.StateOK,
Word: "Connected", Since: now.Add(-60 * 24 * time.Hour), Facts: map[string]any{"type": "async-replication", "version": "6.7.3"}})
}
prefix := map[string]string{"fa-prod-01": "k8s-pvc", "fa-prod-02": "k8s-pvc", "fa-dr-01": "dr-copy", "fa-vdi-01": "vdi"}[arr]
for n := range volumes {
@@ -146,6 +140,16 @@ func newFlashArray(name, token string, k, volumes int) *fakeCluster {
}
put(r)
}
switch arr {
case "fa-vdi-01":
c.flap("ct1", spellVDIUpdate, func(r *report.Workload, since time.Time) {
r.State, r.Word, r.Since = report.StateUpdating, "Updating", since
})
case "fa-dr-01":
c.flap("connection/c-fa-prod-01", spellDRPaths, func(r *report.Workload, since time.Time) {
r.State, r.Word, r.Since = report.StateDegraded, "Connected, but its paths aren't redundant", since
})
}
return c
}
+91
View File
@@ -0,0 +1,91 @@
package main
import "time"
// What goes wrong in the demo, and when (#35). A real fleet is mostly
// fine: a few things stay wrong until someone gets to them, and the rest
// come and go. So the demo has
//
// - three stories, each a cause with its effects:
// 1. fa-prod-01 lost a drive: the array is degraded and its writes are
// slow while the drive's data is rebuilt (all the time);
// 2. checkout errors: every 2 hours fleet-0032, a PostgreSQL replica,
// falls behind, HAProxy takes it out, and the shop's checkout answers
// with errors until it catches up (20 minutes);
// 3. the legacy datacenter: every 4 hours its server stops answering
// Akamai's liveness test, GTM has nowhere to send the legacy property,
// and its CP code fails (30 minutes);
// - a few lasting warnings anyone would have: a failover nobody moved
// back, a weak key, a VM snapshot left behind, a datastore filling, a
// zone on Flexible SSL, a renewal waiting on someone;
// - problems that come and go: a unit that fails and is restarted, a
// container that exits, a check that flaps, a host update that takes
// a while, a late backup, an IBM i job waiting on an answer.
//
// Spells count from the Unix epoch, not from when the simulator started,
// so every part of it agrees on what's happening now, a restarted demo
// picks up where the clock is, and the history the demo starts with can
// be worked out for any time in the past.
// spell is when one problem is happening: for on, once every every, at
// into each period. A zero every means all the time.
type spell struct {
every, at, on time.Duration
}
// always is a problem that's there all the time.
var always = spell{}
func (s spell) active(t time.Time) bool {
if s.every == 0 {
return true
}
p := time.Duration(t.UnixNano()) % s.every
return p >= s.at && p < s.at+s.on
}
// began is when the current spell started, or the last one if it's over;
// a long time ago for a problem that's always there.
func (s spell) began(t time.Time) time.Time {
if s.every == 0 {
return t.Add(-9 * 24 * time.Hour)
}
p := time.Duration(t.UnixNano()) % s.every
start := t.Add(-p + s.at)
if start.After(t) {
start = start.Add(-s.every)
}
return start
}
// The spells, by story. Periods aren't multiples of each other, so the
// mix on screen changes through the day.
var (
// Story 2: checkout errors.
spellReplicaBehind = spell{every: 2 * time.Hour, at: 40 * time.Minute, on: 20 * time.Minute}
// Story 3: the legacy datacenter.
spellLegacyDC = spell{every: 4 * time.Hour, at: 70 * time.Minute, on: 30 * time.Minute}
// Coming and going.
spellRedisFails = spell{every: 110 * time.Minute, at: 20 * time.Minute, on: 12 * time.Minute}
spellAppExits = spell{every: 75 * time.Minute, at: 5 * time.Minute, on: 8 * time.Minute}
spellLegacy502 = spell{every: 140 * time.Minute, at: 30 * time.Minute, on: 15 * time.Minute}
spellLBFlaps = spell{every: 100 * time.Minute, at: 60 * time.Minute, on: 15 * time.Minute}
spellCrashloop = spell{every: 150 * time.Minute, at: 15 * time.Minute, on: 20 * time.Minute}
spellESXiUpdate = spell{every: 4 * time.Hour, at: 10 * time.Minute, on: 40 * time.Minute}
spellESXiMemory = spell{every: 130 * time.Minute, at: 50 * time.Minute, on: 15 * time.Minute}
spellAHVUpdate = spell{every: 270 * time.Minute, at: 100 * time.Minute, on: 40 * time.Minute}
spellPVEHA = spell{every: 150 * time.Minute, at: 90 * time.Minute, on: 20 * time.Minute}
spellDRPaths = spell{every: 3 * time.Hour, at: 130 * time.Minute, on: 30 * time.Minute}
spellBladeEvac = spell{every: 6 * time.Hour, at: 2 * time.Hour, on: 60 * time.Minute}
spellVDIUpdate = spell{every: 5 * time.Hour, at: 4 * time.Hour, on: 45 * time.Minute}
spellInquiry = spell{every: 3 * time.Hour, at: 110 * time.Minute, on: 30 * time.Minute}
spellOrdersDown = spell{every: 200 * time.Minute, at: 80 * time.Minute, on: 25 * time.Minute}
spellVaultEdgeDown = spell{every: 3 * time.Hour, at: 150 * time.Minute, on: 20 * time.Minute}
spellBlocklisted = spell{every: 5 * time.Hour, at: 3 * time.Hour, on: 40 * time.Minute}
spellLateBackup = spell{every: 6 * time.Hour, at: 5 * time.Hour, on: 50 * time.Minute}
spellBackupShrank = spell{every: 6 * time.Hour, at: 2 * time.Hour, on: 50 * time.Minute}
spellSQLAgent = spell{every: 4 * time.Hour, at: 3 * time.Hour, on: 45 * time.Minute}
spellFSCrash = spell{every: 8 * time.Hour, at: 7 * time.Hour, on: time.Hour}
spellActivation = spell{every: 3 * time.Hour, at: 20 * time.Minute, on: 80 * time.Minute}
)
+106
View File
@@ -0,0 +1,106 @@
package main
import (
"testing"
"time"
"github.com/cairnobs/cairnobs/internal/report"
)
func TestSpell(t *testing.T) {
s := spell{every: time.Hour, at: 20 * time.Minute, on: 10 * time.Minute}
day := time.Date(2026, 11, 16, 0, 0, 0, 0, time.UTC)
for _, c := range []struct {
at time.Duration
active bool
began time.Duration
}{
{19 * time.Minute, false, -40 * time.Minute},
{20 * time.Minute, true, 20 * time.Minute},
{29 * time.Minute, true, 20 * time.Minute},
{30 * time.Minute, false, 20 * time.Minute},
{80 * time.Minute, true, 80 * time.Minute},
} {
now := day.Add(c.at)
if s.active(now) != c.active || !s.began(now).Equal(day.Add(c.began)) {
t.Errorf("at %s: active %v began %s", c.at, s.active(now), s.began(now).Sub(day))
}
}
if !always.active(day) || !always.began(day).Before(day) {
t.Error("always")
}
}
// The demo stays calm: across a day, no more than a few of the problems
// that come and go are on at once, and every one of them is on some of
// the time and off the rest.
func TestScenarioIsCalm(t *testing.T) {
spells := map[string]spell{"replica": spellReplicaBehind, "legacy dc": spellLegacyDC, "redis": spellRedisFails, "app": spellAppExits,
"502": spellLegacy502, "lb": spellLBFlaps, "crashloop": spellCrashloop, "esxi update": spellESXiUpdate, "esxi memory": spellESXiMemory,
"ahv": spellAHVUpdate, "pve": spellPVEHA, "dr paths": spellDRPaths, "blade": spellBladeEvac, "vdi": spellVDIUpdate,
"inquiry": spellInquiry, "orders": spellOrdersDown, "vault edge": spellVaultEdgeDown, "blocklist": spellBlocklisted,
"late backup": spellLateBackup, "shrank": spellBackupShrank, "sql agent": spellSQLAgent, "fs crash": spellFSCrash, "activation": spellActivation}
on := map[string]int{}
most, total, steps := 0, 0, 0
day := time.Date(2026, 11, 16, 0, 0, 0, 0, time.UTC)
for m := 0; m < 24*60; m++ {
now := day.Add(time.Duration(m) * time.Minute)
n := 0
for name, s := range spells {
if s.on <= 0 || s.at+s.on > s.every {
t.Fatalf("%s: on %s at %s of %s", name, s.on, s.at, s.every)
}
if s.active(now) {
on[name]++
n++
}
}
most, total, steps = max(most, n), total+n, steps+1
}
for name := range spells {
if on[name] == 0 || on[name] == 24*60 {
t.Errorf("%s is on %d minutes of the day", name, on[name])
}
}
if avg := float64(total) / float64(steps); avg > 4 || most > 9 {
t.Errorf("on at once: %.1f on average, %d at most", avg, most)
}
}
// A flapping record goes wrong during its spell and comes right after; a
// record only the bad snapshot has comes and goes.
func TestFlapSet(t *testing.T) {
c := &fakeCluster{recs: map[string]report.Workload{}, changed: map[string]bool{}}
good := []report.Workload{{ID: "a", State: report.StateOK, Word: "Active"}, {ID: "b", State: report.StateOK, Word: "Fine"}}
bad := []report.Workload{{ID: "a", State: report.StateDown, Word: "Inactive"}, {ID: "b", State: report.StateOK, Word: "Fine"},
{ID: "q", State: report.StateDegraded, Word: "14 jobs waiting"}}
for _, r := range good {
c.recs[r.ID] = r
}
s := spell{every: time.Hour, at: 0, on: 30 * time.Minute}
c.flapSet(s, good, bad)
if len(c.episodes) != 2 {
t.Fatalf("%d episodes, want a and q", len(c.episodes))
}
hour := time.Date(2026, 11, 16, 3, 0, 0, 0, time.UTC)
c.play(hour.Add(5 * time.Minute))
if c.recs["a"].State != report.StateDown || c.recs["q"].ID != "q" {
t.Fatalf("in the spell: %+v %+v", c.recs["a"], c.recs["q"])
}
c.removed = nil
c.play(hour.Add(40 * time.Minute))
if _, there := c.recs["q"]; c.recs["a"].State != report.StateOK || there || len(c.removed) != 1 {
t.Fatalf("after: %+v, q there %v, removed %v", c.recs["a"], there, c.removed)
}
}
// IBM i's flapping is only what the bad snapshot changes, not the whole
// partition.
func TestIBMiFlapsLittle(t *testing.T) {
for k, part := range ibmiPartitions[:2] {
c := newIBMi("fleet-0041", "t", k)
if n := len(c.episodes); n == 0 || n > 4 {
t.Errorf("%s: %d records flap", part, n)
}
}
}
+15 -13
View File
@@ -31,9 +31,11 @@ import (
// - vault-prod: Vault Enterprise, three healthy Raft servers, DR
// replication streaming to its secondary, a license 300 days out, an
// audit device, and a PKI mount (pki_int) with 400 services'
// certificates (some ending soon) and 2,000 short-lived ones.
// - vault-edge: OpenBao, three Raft servers with one failed (no failure
// tolerance left), and an intermediate CA 12 days from its end.
// certificates (one ending within a week, its renewal overdue) and
// 2,000 short-lived ones.
// - vault-edge: OpenBao, three Raft servers, one of which fails now and
// then and leaves no failure tolerance (scenario.go), and an
// intermediate CA 12 days from its end.
//
// They listen on vaultPort and the port after it; -vault's directory
// keeps their certificate and key, so vault.yaml (written there) stays
@@ -74,7 +76,7 @@ func serveVaults(ctx context.Context, dir string) {
newFakeVault("vault-prod", "1.21.4+ent", true, "pki_int", now, 300*24*time.Hour, 400, 2000,
[]fakeRaft{{"v1", "leader", "alive"}, {"v2", "voter", "alive"}, {"v3", "voter", "alive"}}, 1, 18_000),
newFakeVault("vault-edge", "2.4.1", false, "pki_edge", now, 12*24*time.Hour, 40, 0,
[]fakeRaft{{"e1", "leader", "alive"}, {"e2", "voter", "alive"}, {"e3", "voter", "failed"}}, 0, 900),
[]fakeRaft{{"e1", "leader", "alive"}, {"e2", "voter", "alive"}, {"e3", "voter", "alive"}}, 1, 900),
}
var yaml strings.Builder
yaml.WriteString("vault:\n")
@@ -153,14 +155,11 @@ func newFakeVault(name, version string, ent bool, mount string, now time.Time, c
day := 24 * time.Hour
for i := range services {
cn := fmt.Sprintf("svc-%03d.%s.internal", i, strings.TrimPrefix(name, "vault-"))
// Most a long way from their end; a few within the warning's 21
// days, fewer within the critical 7.
// Most a long way from their end; one of vault-prod's within a
// week, its renewal overdue.
left := time.Duration(25+(i*37)%340) * day
switch {
case i%97 == 5:
left = 5 * day
case i%41 == 7:
left = 15 * day
if name == "vault-prod" && i == 102 {
left = 6 * day
}
for k, end := range []time.Duration{left - 30*day, left} { // the one it replaced, then the renewal
x, p := sign(&x509.Certificate{Subject: pkix.Name{CommonName: cn}, DNSNames: []string{cn}, NotBefore: now.Add(end - 365*day), NotAfter: now.Add(end)},
@@ -209,8 +208,11 @@ func (v *fakeVault) ServeHTTP(w http.ResponseWriter, r *http.Request) {
case p == "sys/leader":
out = map[string]any{"ha_enabled": true, "is_self": true, "leader_address": "https://" + v.servers[0].id + "." + v.name + ":8200"}
case p == "sys/storage/raft/autopilot/state":
servers, voters, healthy := map[string]any{}, []string{}, true
servers, voters, healthy, ft := map[string]any{}, []string{}, true, v.ft
for _, s := range v.servers {
if v.name == "vault-edge" && s.id == "e3" && spellVaultEdgeDown.active(now) {
s.node, s.status, ft = "failed", "voter", 0
}
ok := s.node == "alive"
healthy = healthy && ok
sv := map[string]any{"id": s.id, "name": v.name + "-" + s.id, "address": s.id + ":8201", "node_status": s.node, "healthy": ok,
@@ -221,7 +223,7 @@ func (v *fakeVault) ServeHTTP(w http.ResponseWriter, r *http.Request) {
servers[s.id] = sv
voters = append(voters, s.id)
}
out = data(map[string]any{"healthy": healthy, "failure_tolerance": v.ft, "voters": voters, "servers": servers})
out = data(map[string]any{"healthy": healthy, "failure_tolerance": ft, "voters": voters, "servers": servers})
case p == "auth/token/lookup-self":
out = data(map[string]any{"display_name": "token-obs-beacond", "policies": []string{"default", "obs-monitor"}, "ttl": 20 * 3600,
"creation_ttl": 24 * 3600, "renewable": true, "type": "service"})
+8 -9
View File
@@ -30,12 +30,6 @@ func newVSphere(name, token string, vms int) *fakeCluster {
r := report.Workload{Kind: report.KindNode, ID: fmt.Sprintf("host-%d", 1000+n), Name: fmt.Sprintf("esxi-%02d.example.org", n),
State: report.StateOK, Word: "Connected", Since: now.Add(-time.Duration(30+n) * 24 * time.Hour),
Facts: map[string]any{"cpus": 48, "memory_gib": 768.0, "version": "VMware ESXi 8.0.3 build-24414501"}}
switch n {
case 9:
r.State, r.Word, r.Since = report.StateUpdating, "In maintenance mode", now.Add(-35*time.Minute)
case 13:
r.State, r.Word, r.Facts["alarms"] = report.StateDegraded, "vCenter alarm: Host memory usage", "Host memory usage (red)"
}
put(r)
}
for d := 1; d <= 6; d++ {
@@ -62,13 +56,18 @@ func newVSphere(name, token string, vms int) *fakeCluster {
r.State, r.Word = report.StateStopped, "Powered off"
case k == 77:
r.State, r.Word, r.Facts["oldest_snapshot"] = report.StateDegraded, "Running, with a snapshot 23 days old", now.Add(-23*24*time.Hour).Format(time.DateOnly)
case k == 150:
r.State, r.Word = report.StateDown, "Inaccessible: its files can't be reached"
r.Since = now.Add(-9 * time.Minute)
case k%40 == 5:
r.Facts["vmware_tools"] = "not running"
}
put(r)
}
// Patching takes a host into maintenance for a while; a busy one
// raises a memory alarm now and then (scenario.go).
c.flap("host-1009", spellESXiUpdate, func(r *report.Workload, since time.Time) {
r.State, r.Word, r.Since = report.StateUpdating, "In maintenance mode", since
})
c.flap("host-1013", spellESXiMemory, func(r *report.Workload, since time.Time) {
r.State, r.Word, r.Since, r.Facts["alarms"] = report.StateDegraded, "vCenter alarm: Host memory usage", since, "Host memory usage (red)"
})
return c
}
+23 -12
View File
@@ -14,10 +14,11 @@ import (
// agents named win-*, outside the sites of 50:
//
// win-dc01 a domain controller, all well
// win-fs01 a file server that rebooted itself 25 minutes ago (Kernel-Power
// 41, critical) and has a disk with bad blocks
// win-sql01 SQL Server: its Agent service should start by itself and has
// stopped; its certificate ends in a few days
// win-fs01 a file server with a disk going bad: bad blocks, and every
// 8 hours it crashes and comes back (Kernel-Power 41, critical)
// win-sql01 SQL Server: its Agent service should start by itself, and
// now and then stops (scenario.go); its certificate ends in a
// few days
// win-hv01 a Hyper-V host with twelve VMs: one saved, one failed
type winServer struct {
name, os, kernel string
@@ -73,17 +74,26 @@ func newWindowsHost(name string, interval time.Duration) *host {
// them.
func (wh *winHost) stamp(h *host, r *report.Report, now time.Time) {
w := wh.w
crashes, lastCrash := 0, time.Time{}
if w.name == "win-fs01" {
// It went down without shutting down cleanly 25 minutes ago, and
// still has, whenever one looks.
h.start = now.Add(-25 * time.Minute)
r.Uptime = now.Sub(h.start).Seconds()
// It goes down without shutting down cleanly and is straight back:
// up since the last time.
lastCrash = spellFSCrash.began(now)
for t := lastCrash; now.Sub(t) < 24*time.Hour; t = t.Add(-spellFSCrash.every) {
crashes++
}
h.start = lastCrash
r.Uptime = max(0, now.Sub(h.start).Seconds())
}
r.Units = nil
for _, u := range w.units {
un := report.Unit{Name: u + ".service", Load: "loaded", Active: "active", Sub: "running", Since: h.start.UTC().Add(time.Minute).Truncate(time.Second)}
if w.name == "win-sql01" && u == "SQLSERVERAGENT" {
un.Active, un.Sub, un.Since = "failed", "dead", time.Time{}
if b := spellSQLAgent.began(now); spellSQLAgent.active(now) {
un.Active, un.Sub, un.Since = "failed", "dead", time.Time{}
} else if b.After(h.start) { // someone started it again
un.Since = b.Add(spellSQLAgent.on).UTC().Truncate(time.Second)
}
}
r.Units = append(r.Units, un)
}
@@ -94,13 +104,14 @@ func (wh *winHost) stamp(h *host, r *report.Report, now time.Time) {
}
switch w.name {
case "win-fs01":
group("System", "Microsoft-Windows-Kernel-Power", 41, "critical", 1, 24*time.Minute, 24*time.Minute,
first := lastCrash.Add(-time.Duration(crashes-1) * spellFSCrash.every)
group("System", "Microsoft-Windows-Kernel-Power", 41, "critical", crashes, now.Sub(first), now.Sub(h.start),
"The system has rebooted without cleanly shutting down first. This error could be caused if the system stopped responding, crashed, or lost power unexpectedly.")
group("System", "disk", 7, "error", 14, 19*time.Hour, 2*time.Hour, `The device, \Device\Harddisk1\DR1, has a bad block.`)
group("System", "NETLOGON", 5719, "error", 2, 23*time.Minute, 23*time.Minute,
group("System", "NETLOGON", 5719, "error", 2*crashes, now.Sub(first), now.Sub(h.start),
"This computer was not able to set up a secure session with a domain controller in domain CORP due to the following: There are currently no logon servers available to service the logon request.")
case "win-sql01":
group("System", "Service Control Manager", 7034, "error", 3, 6*time.Hour, 40*time.Minute,
group("System", "Service Control Manager", 7034, "error", 3, 6*time.Hour, now.Sub(spellSQLAgent.began(now)),
"The SQL Server Agent (MSSQLSERVER) service terminated unexpectedly. It has done this 3 time(s).")
case "win-hv01":
group("System", "Microsoft-Windows-Hyper-V-Worker", 18560, "critical", 1, 3*time.Hour, 3*time.Hour,