Demo: hosts named for their site and role, collectors on their own #54

Merged
jcoffey-dev merged 1 commits from feature/demo-names into main 2026-11-16 06:29:15 +00:00
21 changed files with 231 additions and 84 deletions

No files matched your search

+3 -3
View File
@@ -605,7 +605,7 @@ configured, and listed them with the rest in 200 ms.
- **Tested** against a fake of the documented API (a TLS server, CA and
pinned fingerprint both; a refused config; quorum lost; a node on its
own), and in the fleet simulator: `-proxmox 400` makes an 8-node
cluster read by fleet-0060, with one guest in HA error, one migrating,
cluster read by phx1-virt01, with one guest in HA error, one migrating,
and hosts 3, 23, 43 … as its VMs. Not against a real Proxmox VE.
### vSphere (step 8, 2026-11-13)
@@ -645,7 +645,7 @@ configured, and listed them with the rest in 200 ms.
certificate, a wrong pin refused, a lapsed session, a VM powered off,
a host into maintenance, a snapshot ten days on), alarms against fixed
states, and in the fleet simulator: `-vsphere 600` makes a vCenter of 16
ESXi hosts read by fleet-0080, with a host in maintenance, one with a
ESXi hosts read by phx1-virt01, with a host in maintenance, one with a
red alarm, a datastore 91% full, an inaccessible VM, an old snapshot,
and hosts 7, 27, 47 … as its VMs, every other one byte-swapped. Not
against a real vCenter.
@@ -689,7 +689,7 @@ configured, and listed them with the rest in 200 ms.
unknown CA refused, a wrong password, a 403 on alerts (a problem on
Prism Central's record) and on VMs (the read fails), and in the fleet
simulator: `-nutanix 450` makes Prism Central pc-01 with three clusters
of six AHV hosts read by fleet-0100, with a host in maintenance, one
of six AHV hosts read by phx1-virt01, with a host in maintenance, one
with a critical alert, a container over its space threshold, and hosts
11, 31, 51 … as its VMs. Not against a real Prism Central.
+1 -1
View File
@@ -218,7 +218,7 @@ results (internal/mapepire's Fake, internal/ibmi's FakePartition), and a
fake HMC built from the answers IBM's own Python library (pypowervm)
keeps from real HMCs (internal/hmc's Fake); its field names were checked
against those answers. Then on a real partition and HMC (question 5).
The demo simulator's `-ibmi 4` adds four partitions read by fleet-0041:
The demo simulator's `-ibmi 4` adds four partitions read by phx1-ibmi01:
prod1 past its system ASP threshold, ha1 a PowerHA standby with its
independent ASP varied off, dev1 on 7.4; and hmc1, their HMC, with the
server they run on, a DR server with its attention light on and a
+1 -1
View File
@@ -95,7 +95,7 @@ PVCs on it going wrong, as one incident.
simulator: `-pure 4` makes fa-prod-01 and -02 (an ActiveCluster pod,
sync and async replication; a failed drive and a volume over its space
on -01), fa-dr-01 (an unbalanced connection) and fa-vdi-01 (a
controller updating), 600 volumes each, all read by fleet-0040. Not
controller updating), 600 volumes each, all read by phx1-stor01. Not
against a real array.
- **Performance:** step 4, below.
+14 -5
View File
@@ -163,10 +163,11 @@ func fleetAkamai(f *akamai.Fake) string {
records += len(recs)
f.SetZone(fmt.Sprintf("zone%02d.example", z), akamai.FakeZone{Records: recs})
}
// Traffic: a long tail, about one request in 2,000 failing.
// Traffic: a long tail, about one request in 2,000 failing, no two
// CP codes alike.
for i := range 320 {
hits := int64(2_000_000 / (i + 1))
f.SetTraffic(int64(400000+i), hits, hits/2000)
hits, errs := cpTraffic(int64(400000 + i))
f.SetTraffic(int64(400000+i), hits, errs)
}
akamaiStory(f, now)
return fmt.Sprintf("%d GTM properties over %d datacenters, 180 CPS enrollments, 320 properties, 40 DNS zones with %d record sets, 320 CP codes", props, sites, records)
@@ -178,6 +179,15 @@ const (
slowActivation = 15 // the property whose production activation drags
)
// cpTraffic is a CP code's requests and errors in a reading: the long
// tail's share for its place in it, give or take 15%, and one in 1,600 to
// 2,600 failing.
func cpTraffic(cp int64) (hits, errs int64) {
k := fmt.Sprint(cp)
hits = int64(float64(2_000_000/(cp-400000+1)) * (0.85 + 0.3*seed(k, "hits")))
return hits, hits / int64(1600+1000*seed(k, "errors"))
}
// slowBase is the slow property as fleetAkamai made it, before its new
// version.
var slowBase akamai.FakePMProperty
@@ -188,8 +198,7 @@ var slowBase akamai.FakePMProperty
func akamaiStory(f *akamai.Fake, now time.Time) {
legacyDown := spellLegacyDC.active(now)
f.SetAlive("corp.example.akadns.net", "legacy", 3100, legacyServer, !legacyDown)
hits := int64(2_000_000 / (legacyCPCode - 400000 + 1))
errs := hits / 2000
hits, errs := cpTraffic(legacyCPCode)
if legacyDown {
errs = hits * 30 / 100
}
+1 -1
View File
@@ -17,7 +17,7 @@ import (
// dozen zones, a load-balanced pool per site whose origins are the sites'
// load balancers, and the last 15 minutes of traffic. obs-beacond reads it
// through cloudflare's stand_in (http on loopback only). Problems on
// purpose: legacy.example still uses Flexible SSL; now and then fleet-0052
// purpose: legacy.example still uses Flexible SSL; now and then iad1-lb02
// fails its pool's health check from eastern North America; and while
// the shop's database replica is behind (scenario.go, story 2) 7% of
// shop.example's requests fail, most on checkout.shop.example.
+134
View File
@@ -0,0 +1,134 @@
package main
import (
"fmt"
"strings"
"sync"
"time"
"github.com/cairnobs/cairnobs/internal/report"
)
// Host names (#40): a site's hosts are named for where they are and what
// they do, "phx1-app07", "iad1-db02", "lon1-kvm01", as a real fleet's
// are, and the agents that read storage, IBM i and virtualization are
// small hosts of their own, not app servers doing it on the side.
// siteCodes are the sites in turn; the fifth is phx2, and so on. Their
// time zones are siteZone's (load.go).
var siteCodes = []string{"phx", "iad", "lon", "fra"}
func siteCode(site int) string {
return fmt.Sprintf("%s%d", siteCodes[site%len(siteCodes)], site/len(siteCodes)+1)
}
// label is what a host's name says it does: its role in the site
// (role), or kvm for a libvirt host.
func label(i int) string {
if isHypervisor(i) {
return "kvm"
}
return map[string]string{"gateway": "gw", "lb": "lb", "app": "app", "db": "db", "cache": "cache", "mq": "mq"}[role(i%siteSize)]
}
// hostName is fleet host i's name: its site, its label and its number
// among the site's hosts with that label.
func hostName(i int) string {
site, pos := i/siteSize, i%siteSize
l, n := label(i), 0
for p := 0; p <= pos; p++ {
if label(site*siteSize+p) == l {
n++
}
}
return fmt.Sprintf("%s-%s%02d", siteCode(site), l, n)
}
var (
indexMu sync.Mutex
indexOf = map[string]int{}
)
// hostIndex is the fleet host a name is, or -1.
func hostIndex(name string) int {
indexMu.Lock()
defer indexMu.Unlock()
if i, ok := indexOf[name]; ok {
return i
}
i := findHost(name)
indexOf[name] = i
return i
}
func findHost(name string) int {
code, _, ok := strings.Cut(name, "-")
if !ok {
return -1
}
site := -1
for s := range 400 {
if siteCode(s) == code {
site = s
break
}
}
if site < 0 {
return -1
}
for p := range siteSize {
if i := site*siteSize + p; hostName(i) == name {
return i
}
}
return -1
}
// collector is an agent on a host of its own that reads things with an
// API instead of an agent: arrays, IBM i, hypervisor managers.
type collector struct {
name string
sources func() []string // what its config lists, read or not, as the agent says
}
var (
storageCollector = collector{"phx1-stor01", func() []string { return pureArrays }}
ibmiCollector = collector{"phx1-ibmi01", func() []string { return append(append([]string{}, ibmiPartitions...), hmcName) }}
virtCollector = collector{"phx1-virt01", func() []string { return []string{"proxmox", vcenterName, prismCentral} }}
collectors = []collector{storageCollector, ibmiCollector, virtCollector}
)
func collectorFor(name string) (collector, bool) {
for _, c := range collectors {
if c.name == name {
return c, true
}
}
return collector{}, false
}
// newCollectorHost is a collector's own host: a small RHEL 9 VM that does
// nothing else.
func newCollectorHost(c collector, interval time.Duration) *host {
d := rhel9
h := &host{start: time.Now().Add(-time.Duration(20+len(c.name)) * 24 * time.Hour)}
h.r = report.Report{
V: report.Version, Agent: "fleet", Host: c.name, Interval: int(interval / time.Second),
Kernel: d.kernel, OS: d.os, CPUs: 4,
Mem: report.Mem{Total: 8 << 30, SwapTotal: 2 << 30, SwapFree: 2 << 30},
Disks: []report.Disk{
{Mount: "/", Device: d.rootDev, FS: d.rootFS, Size: 60 << 30},
{Mount: "/var/lib/cairn-obs", Device: "/dev/vdb", FS: "xfs", Size: 40 << 30},
},
Sources: c.sources(),
Containers: &report.Containers{List: []report.Container{}},
}
return h
}
// isHypervisor says host i runs libvirt: host 1 in every 20 when that's
// an app server's place in its site (a load balancer or a database
// doesn't host VMs on the side).
func isHypervisor(i int) bool { return i%hypervisorEvery == 1 && role(i%siteSize) == "app" }
func isCollector(name string) bool { _, ok := collectorFor(name); return ok }
+2 -2
View File
@@ -19,8 +19,8 @@ import (
// (pgAddr), so obs-beacond can match a member to its API as it does over a
// real network; the status pages come in the hosts' reports, as an agent's
// fetches do.
// Problems on purpose: the leader is fleet-0031, not its usual fleet-0030
// (a failover nobody moved back), and every so often fleet-0032 falls 40
// Problems on purpose: the leader is phx1-db02, not its usual phx1-db01
// (a failover nobody moved back), and every so often phx1-db03 falls 40
// MB behind and HAProxy takes it out of the replicas' pool until it
// catches up (scenario.go, story 2).
const patroniPort = 58420
+9 -5
View File
@@ -6,6 +6,7 @@ import (
"log"
"os"
"path/filepath"
"strings"
"time"
"github.com/cairnobs/cairnobs/internal/report"
@@ -23,9 +24,9 @@ import (
// check); billing.example.com's certificate ends in 9 days;
// the ping to the last site loses 3%; mx2 lands on a block
// list now and then
// units fleet-0041's redis-server fails now and then, and is
// units phx1-cache02's redis-server fails now and then, and is
// restarted
// containers fleet-0007's app-04 exits now and then; fleet-0008's app-09
// containers phx1-app05's app-04 exits now and then; phx1-app06's app-09
// is stopped on purpose
// backups site 1's pg-dump is late now and then; site 2's shrinks to a
// third now and then; site 3's backup disk is 93% full; site
@@ -50,7 +51,10 @@ func role(pos int) string {
// demoUnits are the systemd units host n must keep active.
func demoUnits(n int) []string {
base := []string{"ssh", "chrony"}
base := []string{"ssh", "chrony"} // Debian's and Ubuntu's names
if strings.HasPrefix(distroFor(n).os, "Red Hat") {
base = []string{"sshd", "chronyd"}
}
switch role(n % siteSize) {
case "gateway":
return append(base, "wg-quick@wg0")
@@ -277,10 +281,10 @@ func writeDemo(dir string, hosts, windows int) {
}
}
exp.WriteString(winExpected(windows))
if hosts > ibmiHost {
{
// The IBM i partitions' subsystems that must be active (run -ibmi):
// prod2's ORDERS isn't, and ha1, the standby, expects none.
src := hostName(ibmiHost)
src := ibmiCollector.name
fmt.Fprintf(&exp, "clusters:\n %s:prod1:\n subsystems: [QINTER, QBATCH, QUSRWRK, ORDLIB/ORDERS]\n", src)
fmt.Fprintf(&exp, " %s:prod2:\n subsystems: [QINTER, QBATCH, QUSRWRK, ORDLIB/ORDERS]\n", src)
fmt.Fprintf(&exp, " %s:dev1:\n subsystems: [QINTER, QBATCH]\n", src)
+2 -2
View File
@@ -11,7 +11,7 @@ import (
"github.com/cairnobs/cairnobs/internal/state"
)
// Hosts stay under the warnings (disks 80%, memory 90%), but fleet-0023's
// Hosts stay under the warnings (disks 80%, memory 90%), but phx1-app20's
// log volume, which crosses 80% for a while every ten days (#40).
func TestLoadStaysCalm(t *testing.T) {
now := time.Date(2026, 11, 16, 5, 0, 0, 0, time.UTC)
@@ -37,7 +37,7 @@ func TestLoadStaysCalm(t *testing.T) {
}
}
if over == 0 || over > 30*24/3 {
t.Errorf("fleet-0023's log volume is over 80%% for %d of 720 hours", over)
t.Errorf("phx1-app20's log volume is over 80%% for %d of 720 hours", over)
}
// Not all alike, as every disk at 33% was.
if diskAt(hostName(1), 0, now) == diskAt(hostName(2), 0, now) {
+1 -1
View File
@@ -10,7 +10,7 @@ import (
// The HMC that manages the IBM i partitions' Power servers, as a host
// agent with an hmc: section sends it (docs/ibmi.md, step 4): the same
// agent as the partitions (fleet-0041) reads it, so -ibmi brings it too.
// agent as the partitions (phx1-ibmi01) reads it, so -ibmi brings it too.
// The records are the reader's own, made from a snapshot as the HMC would
// answer.
const hmcName = "hmc1"
+1 -2
View File
@@ -12,10 +12,9 @@ import (
)
// IBM i partitions, as a host agent with an ibmi: section sends them
// (docs/ibmi.md): fleet-0041's agent reads them, each a source of its own.
// (docs/ibmi.md): phx1-ibmi01's agent reads them, each a source of its own.
// The records are the reader's own, made from snapshots as the views
// would answer.
const ibmiHost = 41
// ibmiPartitions are the partitions, in the order -ibmi N takes them.
var ibmiPartitions = []string{"prod1", "prod2", "ha1", "dev1"}
+1 -1
View File
@@ -96,7 +96,7 @@ func memAt(name string, t time.Time) float64 {
// diskAt is how full (0 … 1) a host's disk k is at t: a level of its own,
// growing slowly over a cycle of one to three months until a clean-up
// brings it back. fleet-0023's log volume is the one that fills: from 58%
// brings it back. phx1-app20's log volume is the one that fills: from 58%
// to 86% in ten days, so it warns now and then.
func diskAt(name string, k int, t time.Time) float64 {
what := []string{"root", "data"}[min(k, 1)]
+1 -1
View File
@@ -12,7 +12,7 @@ import (
)
// The demo's mail server (fleet run -mail DIR): a stand-in on loopback for
// the Mail page's round trip, served by fleet-0048. Healthy on purpose,
// the Mail page's round trip, served by phx1-mq04. Healthy on purpose,
// with deliveries taking a few seconds and a small outgoing queue, so the
// page shows what normal looks like.
const mailPort = 58440 // SMTP; IMAP and JMAP on the next two
+43 -36
View File
@@ -85,6 +85,14 @@ func config(args []string) {
fmt.Fprintf(&agents, " - { name: %s, token_sha256: %s, cluster: true }\n", name, hex.EncodeToString(sum[:]))
fmt.Fprintf(&tokens, "%s %s\n", name, tok)
}
for _, c := range collectors {
b := make([]byte, 24)
_, _ = rand.Read(b)
tok := hex.EncodeToString(b)
sum := sha256.Sum256([]byte(tok))
fmt.Fprintf(&agents, " - { name: %s, token_sha256: %s }\n", c.name, hex.EncodeToString(sum[:]))
fmt.Fprintf(&tokens, "%s %s\n", c.name, tok)
}
for _, w := range winServers[:min(max(*windows, 0), len(winServers))] {
b := make([]byte, 24)
_, _ = rand.Read(b)
@@ -99,16 +107,14 @@ func config(args []string) {
if err := os.WriteFile(filepath.Join(*out, "tokens"), tokens.Bytes(), 0o600); err != nil {
log.Fatal(err)
}
fmt.Printf("%d agents, %d cluster agents and %d Windows servers: %s/agents.yaml goes into obs-beacond.yaml; %s/tokens is for fleet run\n",
*n, *clusters, min(max(*windows, 0), len(winServers)), *out, *out)
fmt.Printf("%d agents, %d cluster agents, %d collectors and %d Windows servers: %s/agents.yaml goes into obs-beacond.yaml; %s/tokens is for fleet run\n",
*n, *clusters, len(collectors), min(max(*windows, 0), len(winServers)), *out, *out)
if *demo {
writeDemo(*out, *n, *windows)
fmt.Printf("the demo: %s/demo.yaml has the lines for obs-beacond.yaml; run fleet with -demo and the same -n\n", *out)
}
}
func hostName(i int) string { return fmt.Sprintf("fleet-%04d", i) }
type stats struct {
mu sync.Mutex
lat []time.Duration
@@ -179,11 +185,11 @@ func run(args []string) {
fs.DurationVar(&shape.nodeDown, "node-down", 0, "after this long, cluster-00's node-01 goes down (0: never)")
fs.DurationVar(&shape.nodeDownFor, "node-down-for", time.Minute, "how long node-01 stays down")
fs.BoolVar(&shape.openshift, "openshift", false, "make cluster-01 OpenShift, mid-update, with one operator degraded")
pveGuests := fs.Int("proxmox", 0, "this many VMs and containers in a Proxmox cluster read by fleet-0060's agent (0: none)")
vsphereVMs := fs.Int("vsphere", 0, "this many VMs in a vCenter read by fleet-0080's agent (0: none)")
ntnxVMs := fs.Int("nutanix", 0, "this many VMs in a Prism Central read by fleet-0100's agent (0: none)")
pureN := fs.Int("pure", 0, "this many Pure arrays (up to 5: four FlashArrays with 600 volumes each, then a FlashBlade), all read by fleet-0040's agent (0: none)")
ibmiN := fs.Int("ibmi", 0, "this many IBM i partitions (up to 4), and the HMC of their Power servers, all read by fleet-0041's agent (0: none)")
pveGuests := fs.Int("proxmox", 0, "this many VMs and containers in a Proxmox cluster read by phx1-virt01's agent (0: none)")
vsphereVMs := fs.Int("vsphere", 0, "this many VMs in a vCenter read by phx1-virt01's agent (0: none)")
ntnxVMs := fs.Int("nutanix", 0, "this many VMs in a Prism Central read by phx1-virt01's agent (0: none)")
pureN := fs.Int("pure", 0, "this many Pure arrays (up to 5: four FlashArrays with 600 volumes each, then a FlashBlade), all read by phx1-stor01's agent (0: none)")
ibmiN := fs.Int("ibmi", 0, "this many IBM i partitions (up to 4), and the HMC of their Power servers, all read by phx1-ibmi01's agent (0: none)")
akamaiDir := fs.String("akamai", "", "serve a fake Akamai account on loopback, and write its .edgerc and the akamai: section for obs-beacond.yaml to this directory's akamai.yaml (\"\": none)")
chanDir := fs.String("channels", "", "catch alerts on loopback as ntfy, Discord, PagerDuty, a webhook and an email relay, and write their secrets and the alerts: and reports: sections for obs-beacond.yaml to this directory's channels.yaml (\"\": none)")
mailDir := fs.String("mail", "", "serve a fake mail server on loopback, and write its password and the mail: section for obs-beacond.yaml to this directory's mail.yaml (\"\": none)")
@@ -198,6 +204,7 @@ func run(args []string) {
type agent struct{ host, token string }
var agents, clusterAgents, winAgents []agent
collAgents := map[string]agent{}
f, err := os.Open(*tokFile)
if err != nil {
log.Fatal(err)
@@ -208,6 +215,8 @@ func run(args []string) {
switch {
case ok && strings.HasPrefix(h, "cluster-"):
clusterAgents = append(clusterAgents, agent{h, t})
case ok && isCollector(h):
collAgents[h] = agent{h, t}
case ok && strings.HasPrefix(h, winPrefix):
if _, known := winServerFor(h); known {
winAgents = append(winAgents, agent{h, t})
@@ -283,20 +292,28 @@ func run(args []string) {
var made, delivered, queued atomic.Int64
log.Printf("fleet: %d agents every %s, %d containers each, to %s, for %s", len(agents), *interval, *containers, *servers, *dur)
all := append(slices.Clone(agents), winAgents...)
for _, c := range collectors {
if a, ok := collAgents[c.name]; ok {
all = append(all, a)
}
}
hosts := make([]*host, len(all))
for i, a := range all {
var h *host
c, isColl := collectorFor(a.host)
switch {
case isColl: // collectors.go
h = newCollectorHost(c, *interval)
case i >= len(agents): // a Windows server (windows.go)
h = newWindowsHost(a.host, *interval)
default:
h = newHost(i, a.host, *containers, *interval)
}
if *demo && h.win == nil {
if *demo && h.win == nil && !isColl {
h.demo = newDemoHost(fleetIndex(i, a.host), len(agents), demoAll)
h.demo.setup(&h.r)
}
if *withMap && h.win == nil {
if *withMap && h.win == nil && !isColl {
h.m = fakeMap(i, a.host, h.r.Containers)
h.certs, h.files = hostCerts(i, a.host)
}
@@ -311,30 +328,31 @@ func run(args []string) {
for _, a := range clusterAgents {
sims = append(sims, sim{newCluster(a.host, a.token, shape), a.host == "cluster-00"})
}
if *pveGuests > 0 && len(agents) > proxmoxHost {
a := agents[proxmoxHost]
virt, virtOK := collAgents[virtCollector.name]
if *pveGuests > 0 && virtOK {
a := virt
sims = append(sims, sim{c: newProxmox(a.host, a.token, *pveGuests)})
log.Printf("fleet: Proxmox cluster %s, %d nodes, %d guests, read by %s", proxmoxCluster, proxmoxNodes, *pveGuests, a.host)
}
if *vsphereVMs > 0 && len(agents) > vcenterHost {
a := agents[vcenterHost]
if *vsphereVMs > 0 && virtOK {
a := virt
sims = append(sims, sim{c: newVSphere(a.host, a.token, *vsphereVMs)})
log.Printf("fleet: vCenter %s, %d ESXi hosts, %d VMs, read by %s", vcenterName, esxiHosts, *vsphereVMs, a.host)
}
if *ntnxVMs > 0 && len(agents) > nutanixHost {
a := agents[nutanixHost]
if *ntnxVMs > 0 && virtOK {
a := virt
sims = append(sims, sim{c: newNutanix(a.host, a.token, *ntnxVMs)})
log.Printf("fleet: Prism Central %s, %d clusters of %d AHV hosts, %d VMs, read by %s", prismCentral, ntnxClusters, ahvPerPE, *ntnxVMs, a.host)
}
if n := min(*pureN, len(pureArrays)); n > 0 && len(agents) > pureHost {
a := agents[pureHost]
if a, ok := collAgents[storageCollector.name]; ok && *pureN > 0 {
n := min(*pureN, len(pureArrays))
for k := range n {
sims = append(sims, sim{c: newPure(a.host, a.token, k)})
}
log.Printf("fleet: %d Pure arrays (%v), read by %s", n, pureArrays[:n], a.host)
}
if n := min(*ibmiN, len(ibmiPartitions)); n > 0 && len(agents) > ibmiHost {
a := agents[ibmiHost]
if a, ok := collAgents[ibmiCollector.name]; ok && *ibmiN > 0 {
n := min(*ibmiN, len(ibmiPartitions))
for k := range n {
sims = append(sims, sim{c: newIBMi(a.host, a.token, k)})
}
@@ -584,7 +602,8 @@ type host struct {
certsSent time.Time
}
// hypervisorEvery says which fake hosts run libvirt: host 1 in every 20,
// hypervisorEvery says which fake hosts run libvirt: host 1 in every 20
// that is an app server (isHypervisor),
// with libvirtVMs VMs. Hosts 2, 6, 10, 14 and 18 in the same 20 are its
// first five VMs, with their domain UUIDs as SMBIOS UUIDs.
const (
@@ -688,18 +707,6 @@ func newHost(i int, name string, containers int, interval time.Duration) *host {
},
Sources: []string{}, // what it reads beside itself, as the agent says
}
switch i {
case proxmoxHost:
h.r.Sources = []string{"proxmox"}
case vcenterHost:
h.r.Sources = []string{vcenterName}
case nutanixHost:
h.r.Sources = []string{prismCentral}
case pureHost:
h.r.Sources = pureArrays // what its pure: section lists, read or not
case ibmiHost:
h.r.Sources = append(append([]string{}, ibmiPartitions...), hmcName)
}
list := make([]report.Container, containers)
for c := range list {
list[c] = report.Container{ID: fmt.Sprintf("%012x", mrand.Uint64()), Name: fmt.Sprintf("app-%02d", c), Image: fmt.Sprintf("registry.example.org/app-%02d:1.%d", c, c%7),
@@ -709,10 +716,10 @@ func newHost(i int, name string, containers int, interval time.Duration) *host {
switch {
case i%vmHostEvery == 0:
h.r.Machine = vmUUID("cluster-00", i/vmHostEvery)
case i%hypervisorEvery == 1:
case isHypervisor(i):
h.vms = hypervisorVMs(i, time.Now())
h.r.CPUs, h.r.Mem.Total = 64, 512<<30
case i%vmHostEvery == 2:
case i%vmHostEvery == 2 && isHypervisor(i/hypervisorEvery*hypervisorEvery+1):
h.r.Machine = vmUUID("libvirt-"+hostName(i/hypervisorEvery*hypervisorEvery+1), (i%hypervisorEvery-2)/vmHostEvery)
case i%proxmoxGuestEvery == 3:
h.r.Machine = vmUUID("proxmox-"+proxmoxCluster, i/proxmoxGuestEvery)
+1 -3
View File
@@ -4,8 +4,6 @@ import (
"crypto/sha256"
"encoding/base64"
"fmt"
"strconv"
"strings"
"time"
"github.com/cairnobs/cairnobs/internal/report"
@@ -24,7 +22,7 @@ import (
const siteSize = 50
func fleetIndex(i int, name string) int {
if n, err := strconv.Atoi(strings.TrimPrefix(name, "fleet-")); err == nil {
if n := hostIndex(name); n >= 0 {
return n
}
return i
+2 -3
View File
@@ -8,12 +8,11 @@ import (
)
// A Prism Central, as a host agent with a nutanix: section sends it
// (docs/cluster-watcher.md, step 9): fleet-0100's agent reads it.
// (docs/cluster-watcher.md, step 9): phx1-virt01's agent reads it.
const (
prismCentral = "pc-01.example.org"
ntnxClusters = 3
ahvPerPE = 6 // AHV hosts in each cluster
nutanixHost = 100 // the fleet host whose agent reads it
ahvPerPE = 6 // AHV hosts in each cluster
// nutanixGuestEvery: host 11 in every 20 is Prism Central's VM number i/20.
nutanixGuestEvery = 20
)
-1
View File
@@ -12,7 +12,6 @@ import (
const (
proxmoxCluster = "pve-prod"
proxmoxNodes = 8
proxmoxHost = 60 // the fleet host whose agent reads it
// proxmoxGuestEvery: host 3 in every 20 is the cluster's VM number i/20.
proxmoxGuestEvery = 20
)
+1 -2
View File
@@ -11,9 +11,8 @@ import (
)
// FlashArrays, as a host agent with a pure: section sends them
// (docs/pure.md): fleet-0040's agent reads them all, each a source of its
// (docs/pure.md): phx1-stor01's agent reads them all, each a source of its
// own.
const pureHost = 40 // the fleet host whose agent reads them
// pureArrays are the arrays, in the order -pure N takes them.
var pureArrays = []string{"fa-prod-01", "fa-prod-02", "fa-dr-01", "fa-vdi-01", "fb-prod-01"}
+11 -11
View File
@@ -9,7 +9,7 @@ import "time"
// - three stories, each a cause with its effects:
// 1. fa-prod-01 lost a drive: the array is degraded and its writes are
// slow while the drive's data is rebuilt (all the time);
// 2. checkout errors: every 2 hours fleet-0032, a PostgreSQL replica,
// 2. checkout errors: every 2 hours phx1-db03, a PostgreSQL replica,
// falls behind, HAProxy takes it out, and the shop's checkout answers
// with errors until it catches up (20 minutes);
// 3. the legacy datacenter: every 4 hours its server stops answering
@@ -113,14 +113,14 @@ var pastProblems = []past{
{s: spellReplicaBehind, key: "cloudflare/errors/shop.example", sev: "warning", href: "/cloudflare",
title: "shop.example: 7% of requests failing",
detail: "Most on checkout.shop.example: 503s while HAProxy had one PostgreSQL replica for the shop's reads.",
note: "fleet-0032 fell 40 MB behind during the nightly vacuum and HAProxy took it out; the one replica left couldn't carry checkout's reads. Moving the vacuum out of business hours."},
{s: spellAppExits, key: "container/fleet-0007/app-04", sev: "critical", host: "fleet-0007", href: "/containers",
title: "app-04 on fleet-0007: not running"},
{s: spellDRPaths, key: "cluster/fleet-0040:fa-dr-01/connection/fa-prod-01", sev: "critical", host: "fleet-0040:fa-dr-01", href: "/clusters",
title: "Replication to fa-prod-01 in fa-dr-01 (fleet-0040): connected, but its paths aren't redundant"},
{s: spellLBFlaps, key: "cloudflare/origin/site-1-web/fleet-0052", sev: "warning", host: "fleet-0052", href: "/cloudflare",
title: "fleet-0052 is failing site-1-web's health check"},
{s: spellLBFlaps, key: "akamai/server/api.shop.example.akadns.net/203.0.2.11", sev: "warning", host: "fleet-0101", href: "/akamai",
note: hostName(pgLagging) + " fell 40 MB behind during the nightly vacuum and HAProxy took it out; the one replica left couldn't carry checkout's reads. Moving the vacuum out of business hours."},
{s: spellAppExits, key: "container/" + hostName(demoExited) + "/app-04", sev: "critical", host: hostName(demoExited), href: "/containers",
title: "app-04 on " + hostName(demoExited) + ": not running"},
{s: spellDRPaths, key: "cluster/" + storageCollector.name + ":fa-dr-01/connection/fa-prod-01", sev: "critical", host: storageCollector.name + ":fa-dr-01", href: "/clusters",
title: "Replication to fa-prod-01 in fa-dr-01 (" + storageCollector.name + "): connected, but its paths aren't redundant"},
{s: spellLBFlaps, key: "cloudflare/origin/site-1-web/" + hostName(52), sev: "warning", host: hostName(52), href: "/cloudflare",
title: hostName(52) + " is failing site-1-web's health check"},
{s: spellLBFlaps, key: "akamai/server/api.shop.example.akadns.net/203.0.2.11", sev: "warning", host: hostName(101), href: "/akamai",
title: "203.0.2.11 in Singapore is failing GTM's liveness test for api.shop.example.akadns.net"},
{s: spellVaultEdgeDown, key: "node/obs-beacond:vault-edge/vault-edge-e3", sev: "critical", host: "obs-beacond:vault-edge",
href: "/workloads?source=obs-beacond%3Avault-edge&q=vault-edge-e3", title: "Server vault-edge-e3 in vault-edge (obs-beacond) is down"},
@@ -128,6 +128,6 @@ var pastProblems = []past{
href: "/workloads?source=cluster-00&ns=ns-07&q=app-097", title: "deployment/app-097 in ns-07 (cluster-00): 1 of 10 pods: 1 restarting (CrashLoopBackOff)"},
{s: spellLegacy502, key: "check/web-legacy", sev: "warning", href: "/checks", title: "legacy.example.com is down"},
{s: spellBlocklisted, key: "check/rbl-mx2", sev: "warning", href: "/checks", title: "mx2.example.com on block lists is down"},
{s: spellLateBackup, key: "backup/fleet-0050/received/fleet-0080/pg-dump", sev: "serious", host: "fleet-0050", href: "/backups",
title: "fleet-0080's pg-dump backup on fleet-0050 is 26 hours old"},
{s: spellLateBackup, key: "backup/" + hostName(50) + "/received/" + hostName(80) + "/pg-dump", sev: "serious", host: hostName(50), href: "/backups",
title: hostName(80) + "'s pg-dump backup on " + hostName(50) + " is 26 hours old"},
}
+1 -1
View File
@@ -98,7 +98,7 @@ func TestFlapSet(t *testing.T) {
// partition.
func TestIBMiFlapsLittle(t *testing.T) {
for k, part := range ibmiPartitions[:2] {
c := newIBMi("fleet-0041", "t", k)
c := newIBMi(ibmiCollector.name, "t", k)
if n := len(c.episodes); n == 0 || n > 4 {
t.Errorf("%s: %d records flap", part, n)
}
+1 -2
View File
@@ -8,11 +8,10 @@ import (
)
// A vCenter, as a host agent with a vsphere: section sends it
// (docs/cluster-watcher.md, step 8): fleet-0080's agent reads it.
// (docs/cluster-watcher.md, step 8): phx1-virt01's agent reads it.
const (
vcenterName = "vcsa-01.example.org"
esxiHosts = 16
vcenterHost = 80 // the fleet host whose agent reads it
// vsphereGuestEvery: host 7 in every 20 is the vCenter's VM number
// i/20; every other one on old virtual hardware (its UUID byte-swapped).
vsphereGuestEvery = 20