Demo: hosts named for their site and role, collectors on their own #54
No files matched your search
@@ -605,7 +605,7 @@ configured, and listed them with the rest in 200 ms.
|
||||
- **Tested** against a fake of the documented API (a TLS server, CA and
|
||||
pinned fingerprint both; a refused config; quorum lost; a node on its
|
||||
own), and in the fleet simulator: `-proxmox 400` makes an 8-node
|
||||
cluster read by fleet-0060, with one guest in HA error, one migrating,
|
||||
cluster read by phx1-virt01, with one guest in HA error, one migrating,
|
||||
and hosts 3, 23, 43 … as its VMs. Not against a real Proxmox VE.
|
||||
|
||||
### vSphere (step 8, 2026-11-13)
|
||||
@@ -645,7 +645,7 @@ configured, and listed them with the rest in 200 ms.
|
||||
certificate, a wrong pin refused, a lapsed session, a VM powered off,
|
||||
a host into maintenance, a snapshot ten days on), alarms against fixed
|
||||
states, and in the fleet simulator: `-vsphere 600` makes a vCenter of 16
|
||||
ESXi hosts read by fleet-0080, with a host in maintenance, one with a
|
||||
ESXi hosts read by phx1-virt01, with a host in maintenance, one with a
|
||||
red alarm, a datastore 91% full, an inaccessible VM, an old snapshot,
|
||||
and hosts 7, 27, 47 … as its VMs, every other one byte-swapped. Not
|
||||
against a real vCenter.
|
||||
@@ -689,7 +689,7 @@ configured, and listed them with the rest in 200 ms.
|
||||
unknown CA refused, a wrong password, a 403 on alerts (a problem on
|
||||
Prism Central's record) and on VMs (the read fails), and in the fleet
|
||||
simulator: `-nutanix 450` makes Prism Central pc-01 with three clusters
|
||||
of six AHV hosts read by fleet-0100, with a host in maintenance, one
|
||||
of six AHV hosts read by phx1-virt01, with a host in maintenance, one
|
||||
with a critical alert, a container over its space threshold, and hosts
|
||||
11, 31, 51 … as its VMs. Not against a real Prism Central.
|
||||
|
||||
|
||||
+1
-1
@@ -218,7 +218,7 @@ results (internal/mapepire's Fake, internal/ibmi's FakePartition), and a
|
||||
fake HMC built from the answers IBM's own Python library (pypowervm)
|
||||
keeps from real HMCs (internal/hmc's Fake); its field names were checked
|
||||
against those answers. Then on a real partition and HMC (question 5).
|
||||
The demo simulator's `-ibmi 4` adds four partitions read by fleet-0041:
|
||||
The demo simulator's `-ibmi 4` adds four partitions read by phx1-ibmi01:
|
||||
prod1 past its system ASP threshold, ha1 a PowerHA standby with its
|
||||
independent ASP varied off, dev1 on 7.4; and hmc1, their HMC, with the
|
||||
server they run on, a DR server with its attention light on and a
|
||||
|
||||
+1
-1
@@ -95,7 +95,7 @@ PVCs on it going wrong, as one incident.
|
||||
simulator: `-pure 4` makes fa-prod-01 and -02 (an ActiveCluster pod,
|
||||
sync and async replication; a failed drive and a volume over its space
|
||||
on -01), fa-dr-01 (an unbalanced connection) and fa-vdi-01 (a
|
||||
controller updating), 600 volumes each, all read by fleet-0040. Not
|
||||
controller updating), 600 volumes each, all read by phx1-stor01. Not
|
||||
against a real array.
|
||||
- **Performance:** step 4, below.
|
||||
|
||||
|
||||
+14
-5
@@ -163,10 +163,11 @@ func fleetAkamai(f *akamai.Fake) string {
|
||||
records += len(recs)
|
||||
f.SetZone(fmt.Sprintf("zone%02d.example", z), akamai.FakeZone{Records: recs})
|
||||
}
|
||||
// Traffic: a long tail, about one request in 2,000 failing.
|
||||
// Traffic: a long tail, about one request in 2,000 failing, no two
|
||||
// CP codes alike.
|
||||
for i := range 320 {
|
||||
hits := int64(2_000_000 / (i + 1))
|
||||
f.SetTraffic(int64(400000+i), hits, hits/2000)
|
||||
hits, errs := cpTraffic(int64(400000 + i))
|
||||
f.SetTraffic(int64(400000+i), hits, errs)
|
||||
}
|
||||
akamaiStory(f, now)
|
||||
return fmt.Sprintf("%d GTM properties over %d datacenters, 180 CPS enrollments, 320 properties, 40 DNS zones with %d record sets, 320 CP codes", props, sites, records)
|
||||
@@ -178,6 +179,15 @@ const (
|
||||
slowActivation = 15 // the property whose production activation drags
|
||||
)
|
||||
|
||||
// cpTraffic is a CP code's requests and errors in a reading: the long
|
||||
// tail's share for its place in it, give or take 15%, and one in 1,600 to
|
||||
// 2,600 failing.
|
||||
func cpTraffic(cp int64) (hits, errs int64) {
|
||||
k := fmt.Sprint(cp)
|
||||
hits = int64(float64(2_000_000/(cp-400000+1)) * (0.85 + 0.3*seed(k, "hits")))
|
||||
return hits, hits / int64(1600+1000*seed(k, "errors"))
|
||||
}
|
||||
|
||||
// slowBase is the slow property as fleetAkamai made it, before its new
|
||||
// version.
|
||||
var slowBase akamai.FakePMProperty
|
||||
@@ -188,8 +198,7 @@ var slowBase akamai.FakePMProperty
|
||||
func akamaiStory(f *akamai.Fake, now time.Time) {
|
||||
legacyDown := spellLegacyDC.active(now)
|
||||
f.SetAlive("corp.example.akadns.net", "legacy", 3100, legacyServer, !legacyDown)
|
||||
hits := int64(2_000_000 / (legacyCPCode - 400000 + 1))
|
||||
errs := hits / 2000
|
||||
hits, errs := cpTraffic(legacyCPCode)
|
||||
if legacyDown {
|
||||
errs = hits * 30 / 100
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ import (
|
||||
// dozen zones, a load-balanced pool per site whose origins are the sites'
|
||||
// load balancers, and the last 15 minutes of traffic. obs-beacond reads it
|
||||
// through cloudflare's stand_in (http on loopback only). Problems on
|
||||
// purpose: legacy.example still uses Flexible SSL; now and then fleet-0052
|
||||
// purpose: legacy.example still uses Flexible SSL; now and then iad1-lb02
|
||||
// fails its pool's health check from eastern North America; and while
|
||||
// the shop's database replica is behind (scenario.go, story 2) 7% of
|
||||
// shop.example's requests fail, most on checkout.shop.example.
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/cairnobs/cairnobs/internal/report"
|
||||
)
|
||||
|
||||
// Host names (#40): a site's hosts are named for where they are and what
|
||||
// they do, "phx1-app07", "iad1-db02", "lon1-kvm01", as a real fleet's
|
||||
// are, and the agents that read storage, IBM i and virtualization are
|
||||
// small hosts of their own, not app servers doing it on the side.
|
||||
|
||||
// siteCodes are the sites in turn; the fifth is phx2, and so on. Their
|
||||
// time zones are siteZone's (load.go).
|
||||
var siteCodes = []string{"phx", "iad", "lon", "fra"}
|
||||
|
||||
func siteCode(site int) string {
|
||||
return fmt.Sprintf("%s%d", siteCodes[site%len(siteCodes)], site/len(siteCodes)+1)
|
||||
}
|
||||
|
||||
// label is what a host's name says it does: its role in the site
|
||||
// (role), or kvm for a libvirt host.
|
||||
func label(i int) string {
|
||||
if isHypervisor(i) {
|
||||
return "kvm"
|
||||
}
|
||||
return map[string]string{"gateway": "gw", "lb": "lb", "app": "app", "db": "db", "cache": "cache", "mq": "mq"}[role(i%siteSize)]
|
||||
}
|
||||
|
||||
// hostName is fleet host i's name: its site, its label and its number
|
||||
// among the site's hosts with that label.
|
||||
func hostName(i int) string {
|
||||
site, pos := i/siteSize, i%siteSize
|
||||
l, n := label(i), 0
|
||||
for p := 0; p <= pos; p++ {
|
||||
if label(site*siteSize+p) == l {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("%s-%s%02d", siteCode(site), l, n)
|
||||
}
|
||||
|
||||
var (
|
||||
indexMu sync.Mutex
|
||||
indexOf = map[string]int{}
|
||||
)
|
||||
|
||||
// hostIndex is the fleet host a name is, or -1.
|
||||
func hostIndex(name string) int {
|
||||
indexMu.Lock()
|
||||
defer indexMu.Unlock()
|
||||
if i, ok := indexOf[name]; ok {
|
||||
return i
|
||||
}
|
||||
i := findHost(name)
|
||||
indexOf[name] = i
|
||||
return i
|
||||
}
|
||||
|
||||
func findHost(name string) int {
|
||||
code, _, ok := strings.Cut(name, "-")
|
||||
if !ok {
|
||||
return -1
|
||||
}
|
||||
site := -1
|
||||
for s := range 400 {
|
||||
if siteCode(s) == code {
|
||||
site = s
|
||||
break
|
||||
}
|
||||
}
|
||||
if site < 0 {
|
||||
return -1
|
||||
}
|
||||
for p := range siteSize {
|
||||
if i := site*siteSize + p; hostName(i) == name {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// collector is an agent on a host of its own that reads things with an
|
||||
// API instead of an agent: arrays, IBM i, hypervisor managers.
|
||||
type collector struct {
|
||||
name string
|
||||
sources func() []string // what its config lists, read or not, as the agent says
|
||||
}
|
||||
|
||||
var (
|
||||
storageCollector = collector{"phx1-stor01", func() []string { return pureArrays }}
|
||||
ibmiCollector = collector{"phx1-ibmi01", func() []string { return append(append([]string{}, ibmiPartitions...), hmcName) }}
|
||||
virtCollector = collector{"phx1-virt01", func() []string { return []string{"proxmox", vcenterName, prismCentral} }}
|
||||
collectors = []collector{storageCollector, ibmiCollector, virtCollector}
|
||||
)
|
||||
|
||||
func collectorFor(name string) (collector, bool) {
|
||||
for _, c := range collectors {
|
||||
if c.name == name {
|
||||
return c, true
|
||||
}
|
||||
}
|
||||
return collector{}, false
|
||||
}
|
||||
|
||||
// newCollectorHost is a collector's own host: a small RHEL 9 VM that does
|
||||
// nothing else.
|
||||
func newCollectorHost(c collector, interval time.Duration) *host {
|
||||
d := rhel9
|
||||
h := &host{start: time.Now().Add(-time.Duration(20+len(c.name)) * 24 * time.Hour)}
|
||||
h.r = report.Report{
|
||||
V: report.Version, Agent: "fleet", Host: c.name, Interval: int(interval / time.Second),
|
||||
Kernel: d.kernel, OS: d.os, CPUs: 4,
|
||||
Mem: report.Mem{Total: 8 << 30, SwapTotal: 2 << 30, SwapFree: 2 << 30},
|
||||
Disks: []report.Disk{
|
||||
{Mount: "/", Device: d.rootDev, FS: d.rootFS, Size: 60 << 30},
|
||||
{Mount: "/var/lib/cairn-obs", Device: "/dev/vdb", FS: "xfs", Size: 40 << 30},
|
||||
},
|
||||
Sources: c.sources(),
|
||||
Containers: &report.Containers{List: []report.Container{}},
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// isHypervisor says host i runs libvirt: host 1 in every 20 when that's
|
||||
// an app server's place in its site (a load balancer or a database
|
||||
// doesn't host VMs on the side).
|
||||
func isHypervisor(i int) bool { return i%hypervisorEvery == 1 && role(i%siteSize) == "app" }
|
||||
|
||||
func isCollector(name string) bool { _, ok := collectorFor(name); return ok }
|
||||
@@ -19,8 +19,8 @@ import (
|
||||
// (pgAddr), so obs-beacond can match a member to its API as it does over a
|
||||
// real network; the status pages come in the hosts' reports, as an agent's
|
||||
// fetches do.
|
||||
// Problems on purpose: the leader is fleet-0031, not its usual fleet-0030
|
||||
// (a failover nobody moved back), and every so often fleet-0032 falls 40
|
||||
// Problems on purpose: the leader is phx1-db02, not its usual phx1-db01
|
||||
// (a failover nobody moved back), and every so often phx1-db03 falls 40
|
||||
// MB behind and HAProxy takes it out of the replicas' pool until it
|
||||
// catches up (scenario.go, story 2).
|
||||
const patroniPort = 58420
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/cairnobs/cairnobs/internal/report"
|
||||
@@ -23,9 +24,9 @@ import (
|
||||
// check); billing.example.com's certificate ends in 9 days;
|
||||
// the ping to the last site loses 3%; mx2 lands on a block
|
||||
// list now and then
|
||||
// units fleet-0041's redis-server fails now and then, and is
|
||||
// units phx1-cache02's redis-server fails now and then, and is
|
||||
// restarted
|
||||
// containers fleet-0007's app-04 exits now and then; fleet-0008's app-09
|
||||
// containers phx1-app05's app-04 exits now and then; phx1-app06's app-09
|
||||
// is stopped on purpose
|
||||
// backups site 1's pg-dump is late now and then; site 2's shrinks to a
|
||||
// third now and then; site 3's backup disk is 93% full; site
|
||||
@@ -50,7 +51,10 @@ func role(pos int) string {
|
||||
|
||||
// demoUnits are the systemd units host n must keep active.
|
||||
func demoUnits(n int) []string {
|
||||
base := []string{"ssh", "chrony"}
|
||||
base := []string{"ssh", "chrony"} // Debian's and Ubuntu's names
|
||||
if strings.HasPrefix(distroFor(n).os, "Red Hat") {
|
||||
base = []string{"sshd", "chronyd"}
|
||||
}
|
||||
switch role(n % siteSize) {
|
||||
case "gateway":
|
||||
return append(base, "wg-quick@wg0")
|
||||
@@ -277,10 +281,10 @@ func writeDemo(dir string, hosts, windows int) {
|
||||
}
|
||||
}
|
||||
exp.WriteString(winExpected(windows))
|
||||
if hosts > ibmiHost {
|
||||
{
|
||||
// The IBM i partitions' subsystems that must be active (run -ibmi):
|
||||
// prod2's ORDERS isn't, and ha1, the standby, expects none.
|
||||
src := hostName(ibmiHost)
|
||||
src := ibmiCollector.name
|
||||
fmt.Fprintf(&exp, "clusters:\n %s:prod1:\n subsystems: [QINTER, QBATCH, QUSRWRK, ORDLIB/ORDERS]\n", src)
|
||||
fmt.Fprintf(&exp, " %s:prod2:\n subsystems: [QINTER, QBATCH, QUSRWRK, ORDLIB/ORDERS]\n", src)
|
||||
fmt.Fprintf(&exp, " %s:dev1:\n subsystems: [QINTER, QBATCH]\n", src)
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
"github.com/cairnobs/cairnobs/internal/state"
|
||||
)
|
||||
|
||||
// Hosts stay under the warnings (disks 80%, memory 90%), but fleet-0023's
|
||||
// Hosts stay under the warnings (disks 80%, memory 90%), but phx1-app20's
|
||||
// log volume, which crosses 80% for a while every ten days (#40).
|
||||
func TestLoadStaysCalm(t *testing.T) {
|
||||
now := time.Date(2026, 11, 16, 5, 0, 0, 0, time.UTC)
|
||||
@@ -37,7 +37,7 @@ func TestLoadStaysCalm(t *testing.T) {
|
||||
}
|
||||
}
|
||||
if over == 0 || over > 30*24/3 {
|
||||
t.Errorf("fleet-0023's log volume is over 80%% for %d of 720 hours", over)
|
||||
t.Errorf("phx1-app20's log volume is over 80%% for %d of 720 hours", over)
|
||||
}
|
||||
// Not all alike, as every disk at 33% was.
|
||||
if diskAt(hostName(1), 0, now) == diskAt(hostName(2), 0, now) {
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
|
||||
// The HMC that manages the IBM i partitions' Power servers, as a host
|
||||
// agent with an hmc: section sends it (docs/ibmi.md, step 4): the same
|
||||
// agent as the partitions (fleet-0041) reads it, so -ibmi brings it too.
|
||||
// agent as the partitions (phx1-ibmi01) reads it, so -ibmi brings it too.
|
||||
// The records are the reader's own, made from a snapshot as the HMC would
|
||||
// answer.
|
||||
const hmcName = "hmc1"
|
||||
|
||||
@@ -12,10 +12,9 @@ import (
|
||||
)
|
||||
|
||||
// IBM i partitions, as a host agent with an ibmi: section sends them
|
||||
// (docs/ibmi.md): fleet-0041's agent reads them, each a source of its own.
|
||||
// (docs/ibmi.md): phx1-ibmi01's agent reads them, each a source of its own.
|
||||
// The records are the reader's own, made from snapshots as the views
|
||||
// would answer.
|
||||
const ibmiHost = 41
|
||||
|
||||
// ibmiPartitions are the partitions, in the order -ibmi N takes them.
|
||||
var ibmiPartitions = []string{"prod1", "prod2", "ha1", "dev1"}
|
||||
|
||||
@@ -96,7 +96,7 @@ func memAt(name string, t time.Time) float64 {
|
||||
|
||||
// diskAt is how full (0 … 1) a host's disk k is at t: a level of its own,
|
||||
// growing slowly over a cycle of one to three months until a clean-up
|
||||
// brings it back. fleet-0023's log volume is the one that fills: from 58%
|
||||
// brings it back. phx1-app20's log volume is the one that fills: from 58%
|
||||
// to 86% in ten days, so it warns now and then.
|
||||
func diskAt(name string, k int, t time.Time) float64 {
|
||||
what := []string{"root", "data"}[min(k, 1)]
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
)
|
||||
|
||||
// The demo's mail server (fleet run -mail DIR): a stand-in on loopback for
|
||||
// the Mail page's round trip, served by fleet-0048. Healthy on purpose,
|
||||
// the Mail page's round trip, served by phx1-mq04. Healthy on purpose,
|
||||
// with deliveries taking a few seconds and a small outgoing queue, so the
|
||||
// page shows what normal looks like.
|
||||
const mailPort = 58440 // SMTP; IMAP and JMAP on the next two
|
||||
|
||||
+43
-36
@@ -85,6 +85,14 @@ func config(args []string) {
|
||||
fmt.Fprintf(&agents, " - { name: %s, token_sha256: %s, cluster: true }\n", name, hex.EncodeToString(sum[:]))
|
||||
fmt.Fprintf(&tokens, "%s %s\n", name, tok)
|
||||
}
|
||||
for _, c := range collectors {
|
||||
b := make([]byte, 24)
|
||||
_, _ = rand.Read(b)
|
||||
tok := hex.EncodeToString(b)
|
||||
sum := sha256.Sum256([]byte(tok))
|
||||
fmt.Fprintf(&agents, " - { name: %s, token_sha256: %s }\n", c.name, hex.EncodeToString(sum[:]))
|
||||
fmt.Fprintf(&tokens, "%s %s\n", c.name, tok)
|
||||
}
|
||||
for _, w := range winServers[:min(max(*windows, 0), len(winServers))] {
|
||||
b := make([]byte, 24)
|
||||
_, _ = rand.Read(b)
|
||||
@@ -99,16 +107,14 @@ func config(args []string) {
|
||||
if err := os.WriteFile(filepath.Join(*out, "tokens"), tokens.Bytes(), 0o600); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
fmt.Printf("%d agents, %d cluster agents and %d Windows servers: %s/agents.yaml goes into obs-beacond.yaml; %s/tokens is for fleet run\n",
|
||||
*n, *clusters, min(max(*windows, 0), len(winServers)), *out, *out)
|
||||
fmt.Printf("%d agents, %d cluster agents, %d collectors and %d Windows servers: %s/agents.yaml goes into obs-beacond.yaml; %s/tokens is for fleet run\n",
|
||||
*n, *clusters, len(collectors), min(max(*windows, 0), len(winServers)), *out, *out)
|
||||
if *demo {
|
||||
writeDemo(*out, *n, *windows)
|
||||
fmt.Printf("the demo: %s/demo.yaml has the lines for obs-beacond.yaml; run fleet with -demo and the same -n\n", *out)
|
||||
}
|
||||
}
|
||||
|
||||
func hostName(i int) string { return fmt.Sprintf("fleet-%04d", i) }
|
||||
|
||||
type stats struct {
|
||||
mu sync.Mutex
|
||||
lat []time.Duration
|
||||
@@ -179,11 +185,11 @@ func run(args []string) {
|
||||
fs.DurationVar(&shape.nodeDown, "node-down", 0, "after this long, cluster-00's node-01 goes down (0: never)")
|
||||
fs.DurationVar(&shape.nodeDownFor, "node-down-for", time.Minute, "how long node-01 stays down")
|
||||
fs.BoolVar(&shape.openshift, "openshift", false, "make cluster-01 OpenShift, mid-update, with one operator degraded")
|
||||
pveGuests := fs.Int("proxmox", 0, "this many VMs and containers in a Proxmox cluster read by fleet-0060's agent (0: none)")
|
||||
vsphereVMs := fs.Int("vsphere", 0, "this many VMs in a vCenter read by fleet-0080's agent (0: none)")
|
||||
ntnxVMs := fs.Int("nutanix", 0, "this many VMs in a Prism Central read by fleet-0100's agent (0: none)")
|
||||
pureN := fs.Int("pure", 0, "this many Pure arrays (up to 5: four FlashArrays with 600 volumes each, then a FlashBlade), all read by fleet-0040's agent (0: none)")
|
||||
ibmiN := fs.Int("ibmi", 0, "this many IBM i partitions (up to 4), and the HMC of their Power servers, all read by fleet-0041's agent (0: none)")
|
||||
pveGuests := fs.Int("proxmox", 0, "this many VMs and containers in a Proxmox cluster read by phx1-virt01's agent (0: none)")
|
||||
vsphereVMs := fs.Int("vsphere", 0, "this many VMs in a vCenter read by phx1-virt01's agent (0: none)")
|
||||
ntnxVMs := fs.Int("nutanix", 0, "this many VMs in a Prism Central read by phx1-virt01's agent (0: none)")
|
||||
pureN := fs.Int("pure", 0, "this many Pure arrays (up to 5: four FlashArrays with 600 volumes each, then a FlashBlade), all read by phx1-stor01's agent (0: none)")
|
||||
ibmiN := fs.Int("ibmi", 0, "this many IBM i partitions (up to 4), and the HMC of their Power servers, all read by phx1-ibmi01's agent (0: none)")
|
||||
akamaiDir := fs.String("akamai", "", "serve a fake Akamai account on loopback, and write its .edgerc and the akamai: section for obs-beacond.yaml to this directory's akamai.yaml (\"\": none)")
|
||||
chanDir := fs.String("channels", "", "catch alerts on loopback as ntfy, Discord, PagerDuty, a webhook and an email relay, and write their secrets and the alerts: and reports: sections for obs-beacond.yaml to this directory's channels.yaml (\"\": none)")
|
||||
mailDir := fs.String("mail", "", "serve a fake mail server on loopback, and write its password and the mail: section for obs-beacond.yaml to this directory's mail.yaml (\"\": none)")
|
||||
@@ -198,6 +204,7 @@ func run(args []string) {
|
||||
|
||||
type agent struct{ host, token string }
|
||||
var agents, clusterAgents, winAgents []agent
|
||||
collAgents := map[string]agent{}
|
||||
f, err := os.Open(*tokFile)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
@@ -208,6 +215,8 @@ func run(args []string) {
|
||||
switch {
|
||||
case ok && strings.HasPrefix(h, "cluster-"):
|
||||
clusterAgents = append(clusterAgents, agent{h, t})
|
||||
case ok && isCollector(h):
|
||||
collAgents[h] = agent{h, t}
|
||||
case ok && strings.HasPrefix(h, winPrefix):
|
||||
if _, known := winServerFor(h); known {
|
||||
winAgents = append(winAgents, agent{h, t})
|
||||
@@ -283,20 +292,28 @@ func run(args []string) {
|
||||
var made, delivered, queued atomic.Int64
|
||||
log.Printf("fleet: %d agents every %s, %d containers each, to %s, for %s", len(agents), *interval, *containers, *servers, *dur)
|
||||
all := append(slices.Clone(agents), winAgents...)
|
||||
for _, c := range collectors {
|
||||
if a, ok := collAgents[c.name]; ok {
|
||||
all = append(all, a)
|
||||
}
|
||||
}
|
||||
hosts := make([]*host, len(all))
|
||||
for i, a := range all {
|
||||
var h *host
|
||||
c, isColl := collectorFor(a.host)
|
||||
switch {
|
||||
case isColl: // collectors.go
|
||||
h = newCollectorHost(c, *interval)
|
||||
case i >= len(agents): // a Windows server (windows.go)
|
||||
h = newWindowsHost(a.host, *interval)
|
||||
default:
|
||||
h = newHost(i, a.host, *containers, *interval)
|
||||
}
|
||||
if *demo && h.win == nil {
|
||||
if *demo && h.win == nil && !isColl {
|
||||
h.demo = newDemoHost(fleetIndex(i, a.host), len(agents), demoAll)
|
||||
h.demo.setup(&h.r)
|
||||
}
|
||||
if *withMap && h.win == nil {
|
||||
if *withMap && h.win == nil && !isColl {
|
||||
h.m = fakeMap(i, a.host, h.r.Containers)
|
||||
h.certs, h.files = hostCerts(i, a.host)
|
||||
}
|
||||
@@ -311,30 +328,31 @@ func run(args []string) {
|
||||
for _, a := range clusterAgents {
|
||||
sims = append(sims, sim{newCluster(a.host, a.token, shape), a.host == "cluster-00"})
|
||||
}
|
||||
if *pveGuests > 0 && len(agents) > proxmoxHost {
|
||||
a := agents[proxmoxHost]
|
||||
virt, virtOK := collAgents[virtCollector.name]
|
||||
if *pveGuests > 0 && virtOK {
|
||||
a := virt
|
||||
sims = append(sims, sim{c: newProxmox(a.host, a.token, *pveGuests)})
|
||||
log.Printf("fleet: Proxmox cluster %s, %d nodes, %d guests, read by %s", proxmoxCluster, proxmoxNodes, *pveGuests, a.host)
|
||||
}
|
||||
if *vsphereVMs > 0 && len(agents) > vcenterHost {
|
||||
a := agents[vcenterHost]
|
||||
if *vsphereVMs > 0 && virtOK {
|
||||
a := virt
|
||||
sims = append(sims, sim{c: newVSphere(a.host, a.token, *vsphereVMs)})
|
||||
log.Printf("fleet: vCenter %s, %d ESXi hosts, %d VMs, read by %s", vcenterName, esxiHosts, *vsphereVMs, a.host)
|
||||
}
|
||||
if *ntnxVMs > 0 && len(agents) > nutanixHost {
|
||||
a := agents[nutanixHost]
|
||||
if *ntnxVMs > 0 && virtOK {
|
||||
a := virt
|
||||
sims = append(sims, sim{c: newNutanix(a.host, a.token, *ntnxVMs)})
|
||||
log.Printf("fleet: Prism Central %s, %d clusters of %d AHV hosts, %d VMs, read by %s", prismCentral, ntnxClusters, ahvPerPE, *ntnxVMs, a.host)
|
||||
}
|
||||
if n := min(*pureN, len(pureArrays)); n > 0 && len(agents) > pureHost {
|
||||
a := agents[pureHost]
|
||||
if a, ok := collAgents[storageCollector.name]; ok && *pureN > 0 {
|
||||
n := min(*pureN, len(pureArrays))
|
||||
for k := range n {
|
||||
sims = append(sims, sim{c: newPure(a.host, a.token, k)})
|
||||
}
|
||||
log.Printf("fleet: %d Pure arrays (%v), read by %s", n, pureArrays[:n], a.host)
|
||||
}
|
||||
if n := min(*ibmiN, len(ibmiPartitions)); n > 0 && len(agents) > ibmiHost {
|
||||
a := agents[ibmiHost]
|
||||
if a, ok := collAgents[ibmiCollector.name]; ok && *ibmiN > 0 {
|
||||
n := min(*ibmiN, len(ibmiPartitions))
|
||||
for k := range n {
|
||||
sims = append(sims, sim{c: newIBMi(a.host, a.token, k)})
|
||||
}
|
||||
@@ -584,7 +602,8 @@ type host struct {
|
||||
certsSent time.Time
|
||||
}
|
||||
|
||||
// hypervisorEvery says which fake hosts run libvirt: host 1 in every 20,
|
||||
// hypervisorEvery says which fake hosts run libvirt: host 1 in every 20
|
||||
// that is an app server (isHypervisor),
|
||||
// with libvirtVMs VMs. Hosts 2, 6, 10, 14 and 18 in the same 20 are its
|
||||
// first five VMs, with their domain UUIDs as SMBIOS UUIDs.
|
||||
const (
|
||||
@@ -688,18 +707,6 @@ func newHost(i int, name string, containers int, interval time.Duration) *host {
|
||||
},
|
||||
Sources: []string{}, // what it reads beside itself, as the agent says
|
||||
}
|
||||
switch i {
|
||||
case proxmoxHost:
|
||||
h.r.Sources = []string{"proxmox"}
|
||||
case vcenterHost:
|
||||
h.r.Sources = []string{vcenterName}
|
||||
case nutanixHost:
|
||||
h.r.Sources = []string{prismCentral}
|
||||
case pureHost:
|
||||
h.r.Sources = pureArrays // what its pure: section lists, read or not
|
||||
case ibmiHost:
|
||||
h.r.Sources = append(append([]string{}, ibmiPartitions...), hmcName)
|
||||
}
|
||||
list := make([]report.Container, containers)
|
||||
for c := range list {
|
||||
list[c] = report.Container{ID: fmt.Sprintf("%012x", mrand.Uint64()), Name: fmt.Sprintf("app-%02d", c), Image: fmt.Sprintf("registry.example.org/app-%02d:1.%d", c, c%7),
|
||||
@@ -709,10 +716,10 @@ func newHost(i int, name string, containers int, interval time.Duration) *host {
|
||||
switch {
|
||||
case i%vmHostEvery == 0:
|
||||
h.r.Machine = vmUUID("cluster-00", i/vmHostEvery)
|
||||
case i%hypervisorEvery == 1:
|
||||
case isHypervisor(i):
|
||||
h.vms = hypervisorVMs(i, time.Now())
|
||||
h.r.CPUs, h.r.Mem.Total = 64, 512<<30
|
||||
case i%vmHostEvery == 2:
|
||||
case i%vmHostEvery == 2 && isHypervisor(i/hypervisorEvery*hypervisorEvery+1):
|
||||
h.r.Machine = vmUUID("libvirt-"+hostName(i/hypervisorEvery*hypervisorEvery+1), (i%hypervisorEvery-2)/vmHostEvery)
|
||||
case i%proxmoxGuestEvery == 3:
|
||||
h.r.Machine = vmUUID("proxmox-"+proxmoxCluster, i/proxmoxGuestEvery)
|
||||
|
||||
@@ -4,8 +4,6 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/cairnobs/cairnobs/internal/report"
|
||||
@@ -24,7 +22,7 @@ import (
|
||||
const siteSize = 50
|
||||
|
||||
func fleetIndex(i int, name string) int {
|
||||
if n, err := strconv.Atoi(strings.TrimPrefix(name, "fleet-")); err == nil {
|
||||
if n := hostIndex(name); n >= 0 {
|
||||
return n
|
||||
}
|
||||
return i
|
||||
|
||||
@@ -8,12 +8,11 @@ import (
|
||||
)
|
||||
|
||||
// A Prism Central, as a host agent with a nutanix: section sends it
|
||||
// (docs/cluster-watcher.md, step 9): fleet-0100's agent reads it.
|
||||
// (docs/cluster-watcher.md, step 9): phx1-virt01's agent reads it.
|
||||
const (
|
||||
prismCentral = "pc-01.example.org"
|
||||
ntnxClusters = 3
|
||||
ahvPerPE = 6 // AHV hosts in each cluster
|
||||
nutanixHost = 100 // the fleet host whose agent reads it
|
||||
ahvPerPE = 6 // AHV hosts in each cluster
|
||||
// nutanixGuestEvery: host 11 in every 20 is Prism Central's VM number i/20.
|
||||
nutanixGuestEvery = 20
|
||||
)
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
const (
|
||||
proxmoxCluster = "pve-prod"
|
||||
proxmoxNodes = 8
|
||||
proxmoxHost = 60 // the fleet host whose agent reads it
|
||||
// proxmoxGuestEvery: host 3 in every 20 is the cluster's VM number i/20.
|
||||
proxmoxGuestEvery = 20
|
||||
)
|
||||
|
||||
@@ -11,9 +11,8 @@ import (
|
||||
)
|
||||
|
||||
// FlashArrays, as a host agent with a pure: section sends them
|
||||
// (docs/pure.md): fleet-0040's agent reads them all, each a source of its
|
||||
// (docs/pure.md): phx1-stor01's agent reads them all, each a source of its
|
||||
// own.
|
||||
const pureHost = 40 // the fleet host whose agent reads them
|
||||
|
||||
// pureArrays are the arrays, in the order -pure N takes them.
|
||||
var pureArrays = []string{"fa-prod-01", "fa-prod-02", "fa-dr-01", "fa-vdi-01", "fb-prod-01"}
|
||||
|
||||
+11
-11
@@ -9,7 +9,7 @@ import "time"
|
||||
// - three stories, each a cause with its effects:
|
||||
// 1. fa-prod-01 lost a drive: the array is degraded and its writes are
|
||||
// slow while the drive's data is rebuilt (all the time);
|
||||
// 2. checkout errors: every 2 hours fleet-0032, a PostgreSQL replica,
|
||||
// 2. checkout errors: every 2 hours phx1-db03, a PostgreSQL replica,
|
||||
// falls behind, HAProxy takes it out, and the shop's checkout answers
|
||||
// with errors until it catches up (20 minutes);
|
||||
// 3. the legacy datacenter: every 4 hours its server stops answering
|
||||
@@ -113,14 +113,14 @@ var pastProblems = []past{
|
||||
{s: spellReplicaBehind, key: "cloudflare/errors/shop.example", sev: "warning", href: "/cloudflare",
|
||||
title: "shop.example: 7% of requests failing",
|
||||
detail: "Most on checkout.shop.example: 503s while HAProxy had one PostgreSQL replica for the shop's reads.",
|
||||
note: "fleet-0032 fell 40 MB behind during the nightly vacuum and HAProxy took it out; the one replica left couldn't carry checkout's reads. Moving the vacuum out of business hours."},
|
||||
{s: spellAppExits, key: "container/fleet-0007/app-04", sev: "critical", host: "fleet-0007", href: "/containers",
|
||||
title: "app-04 on fleet-0007: not running"},
|
||||
{s: spellDRPaths, key: "cluster/fleet-0040:fa-dr-01/connection/fa-prod-01", sev: "critical", host: "fleet-0040:fa-dr-01", href: "/clusters",
|
||||
title: "Replication to fa-prod-01 in fa-dr-01 (fleet-0040): connected, but its paths aren't redundant"},
|
||||
{s: spellLBFlaps, key: "cloudflare/origin/site-1-web/fleet-0052", sev: "warning", host: "fleet-0052", href: "/cloudflare",
|
||||
title: "fleet-0052 is failing site-1-web's health check"},
|
||||
{s: spellLBFlaps, key: "akamai/server/api.shop.example.akadns.net/203.0.2.11", sev: "warning", host: "fleet-0101", href: "/akamai",
|
||||
note: hostName(pgLagging) + " fell 40 MB behind during the nightly vacuum and HAProxy took it out; the one replica left couldn't carry checkout's reads. Moving the vacuum out of business hours."},
|
||||
{s: spellAppExits, key: "container/" + hostName(demoExited) + "/app-04", sev: "critical", host: hostName(demoExited), href: "/containers",
|
||||
title: "app-04 on " + hostName(demoExited) + ": not running"},
|
||||
{s: spellDRPaths, key: "cluster/" + storageCollector.name + ":fa-dr-01/connection/fa-prod-01", sev: "critical", host: storageCollector.name + ":fa-dr-01", href: "/clusters",
|
||||
title: "Replication to fa-prod-01 in fa-dr-01 (" + storageCollector.name + "): connected, but its paths aren't redundant"},
|
||||
{s: spellLBFlaps, key: "cloudflare/origin/site-1-web/" + hostName(52), sev: "warning", host: hostName(52), href: "/cloudflare",
|
||||
title: hostName(52) + " is failing site-1-web's health check"},
|
||||
{s: spellLBFlaps, key: "akamai/server/api.shop.example.akadns.net/203.0.2.11", sev: "warning", host: hostName(101), href: "/akamai",
|
||||
title: "203.0.2.11 in Singapore is failing GTM's liveness test for api.shop.example.akadns.net"},
|
||||
{s: spellVaultEdgeDown, key: "node/obs-beacond:vault-edge/vault-edge-e3", sev: "critical", host: "obs-beacond:vault-edge",
|
||||
href: "/workloads?source=obs-beacond%3Avault-edge&q=vault-edge-e3", title: "Server vault-edge-e3 in vault-edge (obs-beacond) is down"},
|
||||
@@ -128,6 +128,6 @@ var pastProblems = []past{
|
||||
href: "/workloads?source=cluster-00&ns=ns-07&q=app-097", title: "deployment/app-097 in ns-07 (cluster-00): 1 of 10 pods: 1 restarting (CrashLoopBackOff)"},
|
||||
{s: spellLegacy502, key: "check/web-legacy", sev: "warning", href: "/checks", title: "legacy.example.com is down"},
|
||||
{s: spellBlocklisted, key: "check/rbl-mx2", sev: "warning", href: "/checks", title: "mx2.example.com on block lists is down"},
|
||||
{s: spellLateBackup, key: "backup/fleet-0050/received/fleet-0080/pg-dump", sev: "serious", host: "fleet-0050", href: "/backups",
|
||||
title: "fleet-0080's pg-dump backup on fleet-0050 is 26 hours old"},
|
||||
{s: spellLateBackup, key: "backup/" + hostName(50) + "/received/" + hostName(80) + "/pg-dump", sev: "serious", host: hostName(50), href: "/backups",
|
||||
title: hostName(80) + "'s pg-dump backup on " + hostName(50) + " is 26 hours old"},
|
||||
}
|
||||
@@ -98,7 +98,7 @@ func TestFlapSet(t *testing.T) {
|
||||
// partition.
|
||||
func TestIBMiFlapsLittle(t *testing.T) {
|
||||
for k, part := range ibmiPartitions[:2] {
|
||||
c := newIBMi("fleet-0041", "t", k)
|
||||
c := newIBMi(ibmiCollector.name, "t", k)
|
||||
if n := len(c.episodes); n == 0 || n > 4 {
|
||||
t.Errorf("%s: %d records flap", part, n)
|
||||
}
|
||||
|
||||
@@ -8,11 +8,10 @@ import (
|
||||
)
|
||||
|
||||
// A vCenter, as a host agent with a vsphere: section sends it
|
||||
// (docs/cluster-watcher.md, step 8): fleet-0080's agent reads it.
|
||||
// (docs/cluster-watcher.md, step 8): phx1-virt01's agent reads it.
|
||||
const (
|
||||
vcenterName = "vcsa-01.example.org"
|
||||
esxiHosts = 16
|
||||
vcenterHost = 80 // the fleet host whose agent reads it
|
||||
// vsphereGuestEvery: host 7 in every 20 is the vCenter's VM number
|
||||
// i/20; every other one on old virtual hardware (its UUID byte-swapped).
|
||||
vsphereGuestEvery = 20
|
||||
|
||||
Reference in new issue
Block a user