diff --git a/docs/cluster-watcher.md b/docs/cluster-watcher.md index 5abb978..1d9b0f8 100644 --- a/docs/cluster-watcher.md +++ b/docs/cluster-watcher.md @@ -605,7 +605,7 @@ configured, and listed them with the rest in 200 ms. - **Tested** against a fake of the documented API (a TLS server, CA and pinned fingerprint both; a refused config; quorum lost; a node on its own), and in the fleet simulator: `-proxmox 400` makes an 8-node - cluster read by fleet-0060, with one guest in HA error, one migrating, + cluster read by phx1-virt01, with one guest in HA error, one migrating, and hosts 3, 23, 43 … as its VMs. Not against a real Proxmox VE. ### vSphere (step 8, 2026-11-13) @@ -645,7 +645,7 @@ configured, and listed them with the rest in 200 ms. certificate, a wrong pin refused, a lapsed session, a VM powered off, a host into maintenance, a snapshot ten days on), alarms against fixed states, and in the fleet simulator: `-vsphere 600` makes a vCenter of 16 - ESXi hosts read by fleet-0080, with a host in maintenance, one with a + ESXi hosts read by phx1-virt01, with a host in maintenance, one with a red alarm, a datastore 91% full, an inaccessible VM, an old snapshot, and hosts 7, 27, 47 … as its VMs, every other one byte-swapped. Not against a real vCenter. @@ -689,7 +689,7 @@ configured, and listed them with the rest in 200 ms. unknown CA refused, a wrong password, a 403 on alerts (a problem on Prism Central's record) and on VMs (the read fails), and in the fleet simulator: `-nutanix 450` makes Prism Central pc-01 with three clusters - of six AHV hosts read by fleet-0100, with a host in maintenance, one + of six AHV hosts read by phx1-virt01, with a host in maintenance, one with a critical alert, a container over its space threshold, and hosts 11, 31, 51 … as its VMs. Not against a real Prism Central. diff --git a/docs/ibmi.md b/docs/ibmi.md index 3bbcfb0..1591a61 100644 --- a/docs/ibmi.md +++ b/docs/ibmi.md @@ -218,7 +218,7 @@ results (internal/mapepire's Fake, internal/ibmi's FakePartition), and a fake HMC built from the answers IBM's own Python library (pypowervm) keeps from real HMCs (internal/hmc's Fake); its field names were checked against those answers. Then on a real partition and HMC (question 5). -The demo simulator's `-ibmi 4` adds four partitions read by fleet-0041: +The demo simulator's `-ibmi 4` adds four partitions read by phx1-ibmi01: prod1 past its system ASP threshold, ha1 a PowerHA standby with its independent ASP varied off, dev1 on 7.4; and hmc1, their HMC, with the server they run on, a DR server with its attention light on and a diff --git a/docs/pure.md b/docs/pure.md index 9618e13..9861d67 100644 --- a/docs/pure.md +++ b/docs/pure.md @@ -95,7 +95,7 @@ PVCs on it going wrong, as one incident. simulator: `-pure 4` makes fa-prod-01 and -02 (an ActiveCluster pod, sync and async replication; a failed drive and a volume over its space on -01), fa-dr-01 (an unbalanced connection) and fa-vdi-01 (a - controller updating), 600 volumes each, all read by fleet-0040. Not + controller updating), 600 volumes each, all read by phx1-stor01. Not against a real array. - **Performance:** step 4, below. diff --git a/scripts/fleet/akamai.go b/scripts/fleet/akamai.go index 8bd3a99..0140f63 100644 --- a/scripts/fleet/akamai.go +++ b/scripts/fleet/akamai.go @@ -163,10 +163,11 @@ func fleetAkamai(f *akamai.Fake) string { records += len(recs) f.SetZone(fmt.Sprintf("zone%02d.example", z), akamai.FakeZone{Records: recs}) } - // Traffic: a long tail, about one request in 2,000 failing. + // Traffic: a long tail, about one request in 2,000 failing, no two + // CP codes alike. for i := range 320 { - hits := int64(2_000_000 / (i + 1)) - f.SetTraffic(int64(400000+i), hits, hits/2000) + hits, errs := cpTraffic(int64(400000 + i)) + f.SetTraffic(int64(400000+i), hits, errs) } akamaiStory(f, now) return fmt.Sprintf("%d GTM properties over %d datacenters, 180 CPS enrollments, 320 properties, 40 DNS zones with %d record sets, 320 CP codes", props, sites, records) @@ -178,6 +179,15 @@ const ( slowActivation = 15 // the property whose production activation drags ) +// cpTraffic is a CP code's requests and errors in a reading: the long +// tail's share for its place in it, give or take 15%, and one in 1,600 to +// 2,600 failing. +func cpTraffic(cp int64) (hits, errs int64) { + k := fmt.Sprint(cp) + hits = int64(float64(2_000_000/(cp-400000+1)) * (0.85 + 0.3*seed(k, "hits"))) + return hits, hits / int64(1600+1000*seed(k, "errors")) +} + // slowBase is the slow property as fleetAkamai made it, before its new // version. var slowBase akamai.FakePMProperty @@ -188,8 +198,7 @@ var slowBase akamai.FakePMProperty func akamaiStory(f *akamai.Fake, now time.Time) { legacyDown := spellLegacyDC.active(now) f.SetAlive("corp.example.akadns.net", "legacy", 3100, legacyServer, !legacyDown) - hits := int64(2_000_000 / (legacyCPCode - 400000 + 1)) - errs := hits / 2000 + hits, errs := cpTraffic(legacyCPCode) if legacyDown { errs = hits * 30 / 100 } diff --git a/scripts/fleet/cloudflare.go b/scripts/fleet/cloudflare.go index 948f1df..8c22896 100644 --- a/scripts/fleet/cloudflare.go +++ b/scripts/fleet/cloudflare.go @@ -17,7 +17,7 @@ import ( // dozen zones, a load-balanced pool per site whose origins are the sites' // load balancers, and the last 15 minutes of traffic. obs-beacond reads it // through cloudflare's stand_in (http on loopback only). Problems on -// purpose: legacy.example still uses Flexible SSL; now and then fleet-0052 +// purpose: legacy.example still uses Flexible SSL; now and then iad1-lb02 // fails its pool's health check from eastern North America; and while // the shop's database replica is behind (scenario.go, story 2) 7% of // shop.example's requests fail, most on checkout.shop.example. diff --git a/scripts/fleet/collectors.go b/scripts/fleet/collectors.go new file mode 100644 index 0000000..dbfc6ba --- /dev/null +++ b/scripts/fleet/collectors.go @@ -0,0 +1,134 @@ +package main + +import ( + "fmt" + "strings" + "sync" + "time" + + "github.com/cairnobs/cairnobs/internal/report" +) + +// Host names (#40): a site's hosts are named for where they are and what +// they do, "phx1-app07", "iad1-db02", "lon1-kvm01", as a real fleet's +// are, and the agents that read storage, IBM i and virtualization are +// small hosts of their own, not app servers doing it on the side. + +// siteCodes are the sites in turn; the fifth is phx2, and so on. Their +// time zones are siteZone's (load.go). +var siteCodes = []string{"phx", "iad", "lon", "fra"} + +func siteCode(site int) string { + return fmt.Sprintf("%s%d", siteCodes[site%len(siteCodes)], site/len(siteCodes)+1) +} + +// label is what a host's name says it does: its role in the site +// (role), or kvm for a libvirt host. +func label(i int) string { + if isHypervisor(i) { + return "kvm" + } + return map[string]string{"gateway": "gw", "lb": "lb", "app": "app", "db": "db", "cache": "cache", "mq": "mq"}[role(i%siteSize)] +} + +// hostName is fleet host i's name: its site, its label and its number +// among the site's hosts with that label. +func hostName(i int) string { + site, pos := i/siteSize, i%siteSize + l, n := label(i), 0 + for p := 0; p <= pos; p++ { + if label(site*siteSize+p) == l { + n++ + } + } + return fmt.Sprintf("%s-%s%02d", siteCode(site), l, n) +} + +var ( + indexMu sync.Mutex + indexOf = map[string]int{} +) + +// hostIndex is the fleet host a name is, or -1. +func hostIndex(name string) int { + indexMu.Lock() + defer indexMu.Unlock() + if i, ok := indexOf[name]; ok { + return i + } + i := findHost(name) + indexOf[name] = i + return i +} + +func findHost(name string) int { + code, _, ok := strings.Cut(name, "-") + if !ok { + return -1 + } + site := -1 + for s := range 400 { + if siteCode(s) == code { + site = s + break + } + } + if site < 0 { + return -1 + } + for p := range siteSize { + if i := site*siteSize + p; hostName(i) == name { + return i + } + } + return -1 +} + +// collector is an agent on a host of its own that reads things with an +// API instead of an agent: arrays, IBM i, hypervisor managers. +type collector struct { + name string + sources func() []string // what its config lists, read or not, as the agent says +} + +var ( + storageCollector = collector{"phx1-stor01", func() []string { return pureArrays }} + ibmiCollector = collector{"phx1-ibmi01", func() []string { return append(append([]string{}, ibmiPartitions...), hmcName) }} + virtCollector = collector{"phx1-virt01", func() []string { return []string{"proxmox", vcenterName, prismCentral} }} + collectors = []collector{storageCollector, ibmiCollector, virtCollector} +) + +func collectorFor(name string) (collector, bool) { + for _, c := range collectors { + if c.name == name { + return c, true + } + } + return collector{}, false +} + +// newCollectorHost is a collector's own host: a small RHEL 9 VM that does +// nothing else. +func newCollectorHost(c collector, interval time.Duration) *host { + d := rhel9 + h := &host{start: time.Now().Add(-time.Duration(20+len(c.name)) * 24 * time.Hour)} + h.r = report.Report{ + V: report.Version, Agent: "fleet", Host: c.name, Interval: int(interval / time.Second), + Kernel: d.kernel, OS: d.os, CPUs: 4, + Mem: report.Mem{Total: 8 << 30, SwapTotal: 2 << 30, SwapFree: 2 << 30}, + Disks: []report.Disk{ + {Mount: "/", Device: d.rootDev, FS: d.rootFS, Size: 60 << 30}, + {Mount: "/var/lib/cairn-obs", Device: "/dev/vdb", FS: "xfs", Size: 40 << 30}, + }, + Sources: c.sources(), + Containers: &report.Containers{List: []report.Container{}}, + } + return h +} + +// isHypervisor says host i runs libvirt: host 1 in every 20 when that's +// an app server's place in its site (a load balancer or a database +// doesn't host VMs on the side). +func isHypervisor(i int) bool { return i%hypervisorEvery == 1 && role(i%siteSize) == "app" } + +func isCollector(name string) bool { _, ok := collectorFor(name); return ok } diff --git a/scripts/fleet/datalayer.go b/scripts/fleet/datalayer.go index e524a9b..c214ee6 100644 --- a/scripts/fleet/datalayer.go +++ b/scripts/fleet/datalayer.go @@ -19,8 +19,8 @@ import ( // (pgAddr), so obs-beacond can match a member to its API as it does over a // real network; the status pages come in the hosts' reports, as an agent's // fetches do. -// Problems on purpose: the leader is fleet-0031, not its usual fleet-0030 -// (a failover nobody moved back), and every so often fleet-0032 falls 40 +// Problems on purpose: the leader is phx1-db02, not its usual phx1-db01 +// (a failover nobody moved back), and every so often phx1-db03 falls 40 // MB behind and HAProxy takes it out of the replicas' pool until it // catches up (scenario.go, story 2). const patroniPort = 58420 diff --git a/scripts/fleet/demo.go b/scripts/fleet/demo.go index 3f7bd92..594408f 100644 --- a/scripts/fleet/demo.go +++ b/scripts/fleet/demo.go @@ -6,6 +6,7 @@ import ( "log" "os" "path/filepath" + "strings" "time" "github.com/cairnobs/cairnobs/internal/report" @@ -23,9 +24,9 @@ import ( // check); billing.example.com's certificate ends in 9 days; // the ping to the last site loses 3%; mx2 lands on a block // list now and then -// units fleet-0041's redis-server fails now and then, and is +// units phx1-cache02's redis-server fails now and then, and is // restarted -// containers fleet-0007's app-04 exits now and then; fleet-0008's app-09 +// containers phx1-app05's app-04 exits now and then; phx1-app06's app-09 // is stopped on purpose // backups site 1's pg-dump is late now and then; site 2's shrinks to a // third now and then; site 3's backup disk is 93% full; site @@ -50,7 +51,10 @@ func role(pos int) string { // demoUnits are the systemd units host n must keep active. func demoUnits(n int) []string { - base := []string{"ssh", "chrony"} + base := []string{"ssh", "chrony"} // Debian's and Ubuntu's names + if strings.HasPrefix(distroFor(n).os, "Red Hat") { + base = []string{"sshd", "chronyd"} + } switch role(n % siteSize) { case "gateway": return append(base, "wg-quick@wg0") @@ -277,10 +281,10 @@ func writeDemo(dir string, hosts, windows int) { } } exp.WriteString(winExpected(windows)) - if hosts > ibmiHost { + { // The IBM i partitions' subsystems that must be active (run -ibmi): // prod2's ORDERS isn't, and ha1, the standby, expects none. - src := hostName(ibmiHost) + src := ibmiCollector.name fmt.Fprintf(&exp, "clusters:\n %s:prod1:\n subsystems: [QINTER, QBATCH, QUSRWRK, ORDLIB/ORDERS]\n", src) fmt.Fprintf(&exp, " %s:prod2:\n subsystems: [QINTER, QBATCH, QUSRWRK, ORDLIB/ORDERS]\n", src) fmt.Fprintf(&exp, " %s:dev1:\n subsystems: [QINTER, QBATCH]\n", src) diff --git a/scripts/fleet/history_test.go b/scripts/fleet/history_test.go index cee0f4c..8aa8f95 100644 --- a/scripts/fleet/history_test.go +++ b/scripts/fleet/history_test.go @@ -11,7 +11,7 @@ import ( "github.com/cairnobs/cairnobs/internal/state" ) -// Hosts stay under the warnings (disks 80%, memory 90%), but fleet-0023's +// Hosts stay under the warnings (disks 80%, memory 90%), but phx1-app20's // log volume, which crosses 80% for a while every ten days (#40). func TestLoadStaysCalm(t *testing.T) { now := time.Date(2026, 11, 16, 5, 0, 0, 0, time.UTC) @@ -37,7 +37,7 @@ func TestLoadStaysCalm(t *testing.T) { } } if over == 0 || over > 30*24/3 { - t.Errorf("fleet-0023's log volume is over 80%% for %d of 720 hours", over) + t.Errorf("phx1-app20's log volume is over 80%% for %d of 720 hours", over) } // Not all alike, as every disk at 33% was. if diskAt(hostName(1), 0, now) == diskAt(hostName(2), 0, now) { diff --git a/scripts/fleet/hmc.go b/scripts/fleet/hmc.go index c41058b..a249111 100644 --- a/scripts/fleet/hmc.go +++ b/scripts/fleet/hmc.go @@ -10,7 +10,7 @@ import ( // The HMC that manages the IBM i partitions' Power servers, as a host // agent with an hmc: section sends it (docs/ibmi.md, step 4): the same -// agent as the partitions (fleet-0041) reads it, so -ibmi brings it too. +// agent as the partitions (phx1-ibmi01) reads it, so -ibmi brings it too. // The records are the reader's own, made from a snapshot as the HMC would // answer. const hmcName = "hmc1" diff --git a/scripts/fleet/ibmi.go b/scripts/fleet/ibmi.go index c931288..17c2880 100644 --- a/scripts/fleet/ibmi.go +++ b/scripts/fleet/ibmi.go @@ -12,10 +12,9 @@ import ( ) // IBM i partitions, as a host agent with an ibmi: section sends them -// (docs/ibmi.md): fleet-0041's agent reads them, each a source of its own. +// (docs/ibmi.md): phx1-ibmi01's agent reads them, each a source of its own. // The records are the reader's own, made from snapshots as the views // would answer. -const ibmiHost = 41 // ibmiPartitions are the partitions, in the order -ibmi N takes them. var ibmiPartitions = []string{"prod1", "prod2", "ha1", "dev1"} diff --git a/scripts/fleet/load.go b/scripts/fleet/load.go index fc4e829..81d5d09 100644 --- a/scripts/fleet/load.go +++ b/scripts/fleet/load.go @@ -96,7 +96,7 @@ func memAt(name string, t time.Time) float64 { // diskAt is how full (0 … 1) a host's disk k is at t: a level of its own, // growing slowly over a cycle of one to three months until a clean-up -// brings it back. fleet-0023's log volume is the one that fills: from 58% +// brings it back. phx1-app20's log volume is the one that fills: from 58% // to 86% in ten days, so it warns now and then. func diskAt(name string, k int, t time.Time) float64 { what := []string{"root", "data"}[min(k, 1)] diff --git a/scripts/fleet/mail.go b/scripts/fleet/mail.go index 1a1d5b8..4e673ba 100644 --- a/scripts/fleet/mail.go +++ b/scripts/fleet/mail.go @@ -12,7 +12,7 @@ import ( ) // The demo's mail server (fleet run -mail DIR): a stand-in on loopback for -// the Mail page's round trip, served by fleet-0048. Healthy on purpose, +// the Mail page's round trip, served by phx1-mq04. Healthy on purpose, // with deliveries taking a few seconds and a small outgoing queue, so the // page shows what normal looks like. const mailPort = 58440 // SMTP; IMAP and JMAP on the next two diff --git a/scripts/fleet/main.go b/scripts/fleet/main.go index 4219e42..9373c19 100644 --- a/scripts/fleet/main.go +++ b/scripts/fleet/main.go @@ -85,6 +85,14 @@ func config(args []string) { fmt.Fprintf(&agents, " - { name: %s, token_sha256: %s, cluster: true }\n", name, hex.EncodeToString(sum[:])) fmt.Fprintf(&tokens, "%s %s\n", name, tok) } + for _, c := range collectors { + b := make([]byte, 24) + _, _ = rand.Read(b) + tok := hex.EncodeToString(b) + sum := sha256.Sum256([]byte(tok)) + fmt.Fprintf(&agents, " - { name: %s, token_sha256: %s }\n", c.name, hex.EncodeToString(sum[:])) + fmt.Fprintf(&tokens, "%s %s\n", c.name, tok) + } for _, w := range winServers[:min(max(*windows, 0), len(winServers))] { b := make([]byte, 24) _, _ = rand.Read(b) @@ -99,16 +107,14 @@ func config(args []string) { if err := os.WriteFile(filepath.Join(*out, "tokens"), tokens.Bytes(), 0o600); err != nil { log.Fatal(err) } - fmt.Printf("%d agents, %d cluster agents and %d Windows servers: %s/agents.yaml goes into obs-beacond.yaml; %s/tokens is for fleet run\n", - *n, *clusters, min(max(*windows, 0), len(winServers)), *out, *out) + fmt.Printf("%d agents, %d cluster agents, %d collectors and %d Windows servers: %s/agents.yaml goes into obs-beacond.yaml; %s/tokens is for fleet run\n", + *n, *clusters, len(collectors), min(max(*windows, 0), len(winServers)), *out, *out) if *demo { writeDemo(*out, *n, *windows) fmt.Printf("the demo: %s/demo.yaml has the lines for obs-beacond.yaml; run fleet with -demo and the same -n\n", *out) } } -func hostName(i int) string { return fmt.Sprintf("fleet-%04d", i) } - type stats struct { mu sync.Mutex lat []time.Duration @@ -179,11 +185,11 @@ func run(args []string) { fs.DurationVar(&shape.nodeDown, "node-down", 0, "after this long, cluster-00's node-01 goes down (0: never)") fs.DurationVar(&shape.nodeDownFor, "node-down-for", time.Minute, "how long node-01 stays down") fs.BoolVar(&shape.openshift, "openshift", false, "make cluster-01 OpenShift, mid-update, with one operator degraded") - pveGuests := fs.Int("proxmox", 0, "this many VMs and containers in a Proxmox cluster read by fleet-0060's agent (0: none)") - vsphereVMs := fs.Int("vsphere", 0, "this many VMs in a vCenter read by fleet-0080's agent (0: none)") - ntnxVMs := fs.Int("nutanix", 0, "this many VMs in a Prism Central read by fleet-0100's agent (0: none)") - pureN := fs.Int("pure", 0, "this many Pure arrays (up to 5: four FlashArrays with 600 volumes each, then a FlashBlade), all read by fleet-0040's agent (0: none)") - ibmiN := fs.Int("ibmi", 0, "this many IBM i partitions (up to 4), and the HMC of their Power servers, all read by fleet-0041's agent (0: none)") + pveGuests := fs.Int("proxmox", 0, "this many VMs and containers in a Proxmox cluster read by phx1-virt01's agent (0: none)") + vsphereVMs := fs.Int("vsphere", 0, "this many VMs in a vCenter read by phx1-virt01's agent (0: none)") + ntnxVMs := fs.Int("nutanix", 0, "this many VMs in a Prism Central read by phx1-virt01's agent (0: none)") + pureN := fs.Int("pure", 0, "this many Pure arrays (up to 5: four FlashArrays with 600 volumes each, then a FlashBlade), all read by phx1-stor01's agent (0: none)") + ibmiN := fs.Int("ibmi", 0, "this many IBM i partitions (up to 4), and the HMC of their Power servers, all read by phx1-ibmi01's agent (0: none)") akamaiDir := fs.String("akamai", "", "serve a fake Akamai account on loopback, and write its .edgerc and the akamai: section for obs-beacond.yaml to this directory's akamai.yaml (\"\": none)") chanDir := fs.String("channels", "", "catch alerts on loopback as ntfy, Discord, PagerDuty, a webhook and an email relay, and write their secrets and the alerts: and reports: sections for obs-beacond.yaml to this directory's channels.yaml (\"\": none)") mailDir := fs.String("mail", "", "serve a fake mail server on loopback, and write its password and the mail: section for obs-beacond.yaml to this directory's mail.yaml (\"\": none)") @@ -198,6 +204,7 @@ func run(args []string) { type agent struct{ host, token string } var agents, clusterAgents, winAgents []agent + collAgents := map[string]agent{} f, err := os.Open(*tokFile) if err != nil { log.Fatal(err) @@ -208,6 +215,8 @@ func run(args []string) { switch { case ok && strings.HasPrefix(h, "cluster-"): clusterAgents = append(clusterAgents, agent{h, t}) + case ok && isCollector(h): + collAgents[h] = agent{h, t} case ok && strings.HasPrefix(h, winPrefix): if _, known := winServerFor(h); known { winAgents = append(winAgents, agent{h, t}) @@ -283,20 +292,28 @@ func run(args []string) { var made, delivered, queued atomic.Int64 log.Printf("fleet: %d agents every %s, %d containers each, to %s, for %s", len(agents), *interval, *containers, *servers, *dur) all := append(slices.Clone(agents), winAgents...) + for _, c := range collectors { + if a, ok := collAgents[c.name]; ok { + all = append(all, a) + } + } hosts := make([]*host, len(all)) for i, a := range all { var h *host + c, isColl := collectorFor(a.host) switch { + case isColl: // collectors.go + h = newCollectorHost(c, *interval) case i >= len(agents): // a Windows server (windows.go) h = newWindowsHost(a.host, *interval) default: h = newHost(i, a.host, *containers, *interval) } - if *demo && h.win == nil { + if *demo && h.win == nil && !isColl { h.demo = newDemoHost(fleetIndex(i, a.host), len(agents), demoAll) h.demo.setup(&h.r) } - if *withMap && h.win == nil { + if *withMap && h.win == nil && !isColl { h.m = fakeMap(i, a.host, h.r.Containers) h.certs, h.files = hostCerts(i, a.host) } @@ -311,30 +328,31 @@ func run(args []string) { for _, a := range clusterAgents { sims = append(sims, sim{newCluster(a.host, a.token, shape), a.host == "cluster-00"}) } - if *pveGuests > 0 && len(agents) > proxmoxHost { - a := agents[proxmoxHost] + virt, virtOK := collAgents[virtCollector.name] + if *pveGuests > 0 && virtOK { + a := virt sims = append(sims, sim{c: newProxmox(a.host, a.token, *pveGuests)}) log.Printf("fleet: Proxmox cluster %s, %d nodes, %d guests, read by %s", proxmoxCluster, proxmoxNodes, *pveGuests, a.host) } - if *vsphereVMs > 0 && len(agents) > vcenterHost { - a := agents[vcenterHost] + if *vsphereVMs > 0 && virtOK { + a := virt sims = append(sims, sim{c: newVSphere(a.host, a.token, *vsphereVMs)}) log.Printf("fleet: vCenter %s, %d ESXi hosts, %d VMs, read by %s", vcenterName, esxiHosts, *vsphereVMs, a.host) } - if *ntnxVMs > 0 && len(agents) > nutanixHost { - a := agents[nutanixHost] + if *ntnxVMs > 0 && virtOK { + a := virt sims = append(sims, sim{c: newNutanix(a.host, a.token, *ntnxVMs)}) log.Printf("fleet: Prism Central %s, %d clusters of %d AHV hosts, %d VMs, read by %s", prismCentral, ntnxClusters, ahvPerPE, *ntnxVMs, a.host) } - if n := min(*pureN, len(pureArrays)); n > 0 && len(agents) > pureHost { - a := agents[pureHost] + if a, ok := collAgents[storageCollector.name]; ok && *pureN > 0 { + n := min(*pureN, len(pureArrays)) for k := range n { sims = append(sims, sim{c: newPure(a.host, a.token, k)}) } log.Printf("fleet: %d Pure arrays (%v), read by %s", n, pureArrays[:n], a.host) } - if n := min(*ibmiN, len(ibmiPartitions)); n > 0 && len(agents) > ibmiHost { - a := agents[ibmiHost] + if a, ok := collAgents[ibmiCollector.name]; ok && *ibmiN > 0 { + n := min(*ibmiN, len(ibmiPartitions)) for k := range n { sims = append(sims, sim{c: newIBMi(a.host, a.token, k)}) } @@ -584,7 +602,8 @@ type host struct { certsSent time.Time } -// hypervisorEvery says which fake hosts run libvirt: host 1 in every 20, +// hypervisorEvery says which fake hosts run libvirt: host 1 in every 20 +// that is an app server (isHypervisor), // with libvirtVMs VMs. Hosts 2, 6, 10, 14 and 18 in the same 20 are its // first five VMs, with their domain UUIDs as SMBIOS UUIDs. const ( @@ -688,18 +707,6 @@ func newHost(i int, name string, containers int, interval time.Duration) *host { }, Sources: []string{}, // what it reads beside itself, as the agent says } - switch i { - case proxmoxHost: - h.r.Sources = []string{"proxmox"} - case vcenterHost: - h.r.Sources = []string{vcenterName} - case nutanixHost: - h.r.Sources = []string{prismCentral} - case pureHost: - h.r.Sources = pureArrays // what its pure: section lists, read or not - case ibmiHost: - h.r.Sources = append(append([]string{}, ibmiPartitions...), hmcName) - } list := make([]report.Container, containers) for c := range list { list[c] = report.Container{ID: fmt.Sprintf("%012x", mrand.Uint64()), Name: fmt.Sprintf("app-%02d", c), Image: fmt.Sprintf("registry.example.org/app-%02d:1.%d", c, c%7), @@ -709,10 +716,10 @@ func newHost(i int, name string, containers int, interval time.Duration) *host { switch { case i%vmHostEvery == 0: h.r.Machine = vmUUID("cluster-00", i/vmHostEvery) - case i%hypervisorEvery == 1: + case isHypervisor(i): h.vms = hypervisorVMs(i, time.Now()) h.r.CPUs, h.r.Mem.Total = 64, 512<<30 - case i%vmHostEvery == 2: + case i%vmHostEvery == 2 && isHypervisor(i/hypervisorEvery*hypervisorEvery+1): h.r.Machine = vmUUID("libvirt-"+hostName(i/hypervisorEvery*hypervisorEvery+1), (i%hypervisorEvery-2)/vmHostEvery) case i%proxmoxGuestEvery == 3: h.r.Machine = vmUUID("proxmox-"+proxmoxCluster, i/proxmoxGuestEvery) diff --git a/scripts/fleet/netmap.go b/scripts/fleet/netmap.go index c97799c..85d9f4b 100644 --- a/scripts/fleet/netmap.go +++ b/scripts/fleet/netmap.go @@ -4,8 +4,6 @@ import ( "crypto/sha256" "encoding/base64" "fmt" - "strconv" - "strings" "time" "github.com/cairnobs/cairnobs/internal/report" @@ -24,7 +22,7 @@ import ( const siteSize = 50 func fleetIndex(i int, name string) int { - if n, err := strconv.Atoi(strings.TrimPrefix(name, "fleet-")); err == nil { + if n := hostIndex(name); n >= 0 { return n } return i diff --git a/scripts/fleet/nutanix.go b/scripts/fleet/nutanix.go index a8d6e8f..5192e23 100644 --- a/scripts/fleet/nutanix.go +++ b/scripts/fleet/nutanix.go @@ -8,12 +8,11 @@ import ( ) // A Prism Central, as a host agent with a nutanix: section sends it -// (docs/cluster-watcher.md, step 9): fleet-0100's agent reads it. +// (docs/cluster-watcher.md, step 9): phx1-virt01's agent reads it. const ( prismCentral = "pc-01.example.org" ntnxClusters = 3 - ahvPerPE = 6 // AHV hosts in each cluster - nutanixHost = 100 // the fleet host whose agent reads it + ahvPerPE = 6 // AHV hosts in each cluster // nutanixGuestEvery: host 11 in every 20 is Prism Central's VM number i/20. nutanixGuestEvery = 20 ) diff --git a/scripts/fleet/proxmox.go b/scripts/fleet/proxmox.go index b24d4d4..8d2c53e 100644 --- a/scripts/fleet/proxmox.go +++ b/scripts/fleet/proxmox.go @@ -12,7 +12,6 @@ import ( const ( proxmoxCluster = "pve-prod" proxmoxNodes = 8 - proxmoxHost = 60 // the fleet host whose agent reads it // proxmoxGuestEvery: host 3 in every 20 is the cluster's VM number i/20. proxmoxGuestEvery = 20 ) diff --git a/scripts/fleet/pure.go b/scripts/fleet/pure.go index 776e8ac..4e0b206 100644 --- a/scripts/fleet/pure.go +++ b/scripts/fleet/pure.go @@ -11,9 +11,8 @@ import ( ) // FlashArrays, as a host agent with a pure: section sends them -// (docs/pure.md): fleet-0040's agent reads them all, each a source of its +// (docs/pure.md): phx1-stor01's agent reads them all, each a source of its // own. -const pureHost = 40 // the fleet host whose agent reads them // pureArrays are the arrays, in the order -pure N takes them. var pureArrays = []string{"fa-prod-01", "fa-prod-02", "fa-dr-01", "fa-vdi-01", "fb-prod-01"} diff --git a/scripts/fleet/scenario.go b/scripts/fleet/scenario.go index 30f810f..2f2a823 100644 --- a/scripts/fleet/scenario.go +++ b/scripts/fleet/scenario.go @@ -9,7 +9,7 @@ import "time" // - three stories, each a cause with its effects: // 1. fa-prod-01 lost a drive: the array is degraded and its writes are // slow while the drive's data is rebuilt (all the time); -// 2. checkout errors: every 2 hours fleet-0032, a PostgreSQL replica, +// 2. checkout errors: every 2 hours phx1-db03, a PostgreSQL replica, // falls behind, HAProxy takes it out, and the shop's checkout answers // with errors until it catches up (20 minutes); // 3. the legacy datacenter: every 4 hours its server stops answering @@ -113,14 +113,14 @@ var pastProblems = []past{ {s: spellReplicaBehind, key: "cloudflare/errors/shop.example", sev: "warning", href: "/cloudflare", title: "shop.example: 7% of requests failing", detail: "Most on checkout.shop.example: 503s while HAProxy had one PostgreSQL replica for the shop's reads.", - note: "fleet-0032 fell 40 MB behind during the nightly vacuum and HAProxy took it out; the one replica left couldn't carry checkout's reads. Moving the vacuum out of business hours."}, - {s: spellAppExits, key: "container/fleet-0007/app-04", sev: "critical", host: "fleet-0007", href: "/containers", - title: "app-04 on fleet-0007: not running"}, - {s: spellDRPaths, key: "cluster/fleet-0040:fa-dr-01/connection/fa-prod-01", sev: "critical", host: "fleet-0040:fa-dr-01", href: "/clusters", - title: "Replication to fa-prod-01 in fa-dr-01 (fleet-0040): connected, but its paths aren't redundant"}, - {s: spellLBFlaps, key: "cloudflare/origin/site-1-web/fleet-0052", sev: "warning", host: "fleet-0052", href: "/cloudflare", - title: "fleet-0052 is failing site-1-web's health check"}, - {s: spellLBFlaps, key: "akamai/server/api.shop.example.akadns.net/203.0.2.11", sev: "warning", host: "fleet-0101", href: "/akamai", + note: hostName(pgLagging) + " fell 40 MB behind during the nightly vacuum and HAProxy took it out; the one replica left couldn't carry checkout's reads. Moving the vacuum out of business hours."}, + {s: spellAppExits, key: "container/" + hostName(demoExited) + "/app-04", sev: "critical", host: hostName(demoExited), href: "/containers", + title: "app-04 on " + hostName(demoExited) + ": not running"}, + {s: spellDRPaths, key: "cluster/" + storageCollector.name + ":fa-dr-01/connection/fa-prod-01", sev: "critical", host: storageCollector.name + ":fa-dr-01", href: "/clusters", + title: "Replication to fa-prod-01 in fa-dr-01 (" + storageCollector.name + "): connected, but its paths aren't redundant"}, + {s: spellLBFlaps, key: "cloudflare/origin/site-1-web/" + hostName(52), sev: "warning", host: hostName(52), href: "/cloudflare", + title: hostName(52) + " is failing site-1-web's health check"}, + {s: spellLBFlaps, key: "akamai/server/api.shop.example.akadns.net/203.0.2.11", sev: "warning", host: hostName(101), href: "/akamai", title: "203.0.2.11 in Singapore is failing GTM's liveness test for api.shop.example.akadns.net"}, {s: spellVaultEdgeDown, key: "node/obs-beacond:vault-edge/vault-edge-e3", sev: "critical", host: "obs-beacond:vault-edge", href: "/workloads?source=obs-beacond%3Avault-edge&q=vault-edge-e3", title: "Server vault-edge-e3 in vault-edge (obs-beacond) is down"}, @@ -128,6 +128,6 @@ var pastProblems = []past{ href: "/workloads?source=cluster-00&ns=ns-07&q=app-097", title: "deployment/app-097 in ns-07 (cluster-00): 1 of 10 pods: 1 restarting (CrashLoopBackOff)"}, {s: spellLegacy502, key: "check/web-legacy", sev: "warning", href: "/checks", title: "legacy.example.com is down"}, {s: spellBlocklisted, key: "check/rbl-mx2", sev: "warning", href: "/checks", title: "mx2.example.com on block lists is down"}, - {s: spellLateBackup, key: "backup/fleet-0050/received/fleet-0080/pg-dump", sev: "serious", host: "fleet-0050", href: "/backups", - title: "fleet-0080's pg-dump backup on fleet-0050 is 26 hours old"}, + {s: spellLateBackup, key: "backup/" + hostName(50) + "/received/" + hostName(80) + "/pg-dump", sev: "serious", host: hostName(50), href: "/backups", + title: hostName(80) + "'s pg-dump backup on " + hostName(50) + " is 26 hours old"}, } diff --git a/scripts/fleet/scenario_test.go b/scripts/fleet/scenario_test.go index dbbc14f..6a1cb57 100644 --- a/scripts/fleet/scenario_test.go +++ b/scripts/fleet/scenario_test.go @@ -98,7 +98,7 @@ func TestFlapSet(t *testing.T) { // partition. func TestIBMiFlapsLittle(t *testing.T) { for k, part := range ibmiPartitions[:2] { - c := newIBMi("fleet-0041", "t", k) + c := newIBMi(ibmiCollector.name, "t", k) if n := len(c.episodes); n == 0 || n > 4 { t.Errorf("%s: %d records flap", part, n) } diff --git a/scripts/fleet/vsphere.go b/scripts/fleet/vsphere.go index 055f3a3..ec12b54 100644 --- a/scripts/fleet/vsphere.go +++ b/scripts/fleet/vsphere.go @@ -8,11 +8,10 @@ import ( ) // A vCenter, as a host agent with a vsphere: section sends it -// (docs/cluster-watcher.md, step 8): fleet-0080's agent reads it. +// (docs/cluster-watcher.md, step 8): phx1-virt01's agent reads it. const ( vcenterName = "vcsa-01.example.org" esxiHosts = 16 - vcenterHost = 80 // the fleet host whose agent reads it // vsphereGuestEvery: host 7 in every 20 is the vCenter's VM number // i/20; every other one on old virtual hardware (its UUID byte-swapped). vsphereGuestEvery = 20