// fleet is a synthetic fleet for the scale tests // (docs/scale-and-availability.md, How it will be tested): many fake agents // reporting to obs-beacond at the usual interval, with a browser's worth of // interface requests on the side, timing both. // // go run ./scripts/fleet config -n 1000 -clusters 3 -out dir agents for obs-beacond.yaml, and their tokens // go run ./scripts/fleet run -server http://127.0.0.1:5404,http://127.0.0.1:5405 -tokens dir/tokens -for 5m // // Cluster agents (named cluster-NN) play a cluster each (cluster.go). // Agents named win-* are Windows servers (windows.go). // // It's a test tool: tokens go over plain HTTP, and the interface requests // sign in with dev_signin, so point it only at a loopback test server. package main import ( "bufio" "bytes" "context" "crypto/rand" "crypto/sha256" "encoding/hex" "encoding/json" "flag" "fmt" "hash/fnv" "io" "log" "math" mrand "math/rand/v2" "net/http" "net/url" "os" "os/signal" "path/filepath" "slices" "strings" "sync" "sync/atomic" "time" "github.com/cairnobs/cairnobs/internal/netmap" "github.com/cairnobs/cairnobs/internal/report" ) func main() { log.SetFlags(0) if len(os.Args) < 2 { log.Fatal("usage: fleet config|run [flags]") } switch os.Args[1] { case "config": config(os.Args[2:]) case "run": run(os.Args[2:]) default: log.Fatalf("fleet: unknown command %q", os.Args[1]) } } func config(args []string) { fs := flag.NewFlagSet("config", flag.ExitOnError) n := fs.Int("n", 1000, "agents") clusters := fs.Int("clusters", 0, "cluster agents") windows := fs.Int("windows", 0, "Windows servers, up to four (windows.go)") out := fs.String("out", ".", "where to write agents.yaml and tokens") demo := fs.Bool("demo", false, "also write the demo's checks.yaml, expected.yaml, backups.yaml, and demo.yaml with the lines for obs-beacond.yaml (demo.go)") _ = fs.Parse(args) var agents, tokens bytes.Buffer agents.WriteString("agents:\n") for i := range *n { b := make([]byte, 24) _, _ = rand.Read(b) tok := hex.EncodeToString(b) sum := sha256.Sum256([]byte(tok)) fmt.Fprintf(&agents, " - { name: %s, token_sha256: %s }\n", hostName(i), hex.EncodeToString(sum[:])) fmt.Fprintf(&tokens, "%s %s\n", hostName(i), tok) } for i := range *clusters { b := make([]byte, 24) _, _ = rand.Read(b) tok := hex.EncodeToString(b) sum := sha256.Sum256([]byte(tok)) name := fmt.Sprintf("cluster-%02d", i) fmt.Fprintf(&agents, " - { name: %s, token_sha256: %s, cluster: true }\n", name, hex.EncodeToString(sum[:])) fmt.Fprintf(&tokens, "%s %s\n", name, tok) } for _, w := range winServers[:min(max(*windows, 0), len(winServers))] { b := make([]byte, 24) _, _ = rand.Read(b) tok := hex.EncodeToString(b) sum := sha256.Sum256([]byte(tok)) fmt.Fprintf(&agents, " - { name: %s, token_sha256: %s }\n", w.name, hex.EncodeToString(sum[:])) fmt.Fprintf(&tokens, "%s %s\n", w.name, tok) } if err := os.WriteFile(filepath.Join(*out, "agents.yaml"), agents.Bytes(), 0o600); err != nil { log.Fatal(err) } if err := os.WriteFile(filepath.Join(*out, "tokens"), tokens.Bytes(), 0o600); err != nil { log.Fatal(err) } fmt.Printf("%d agents, %d cluster agents and %d Windows servers: %s/agents.yaml goes into obs-beacond.yaml; %s/tokens is for fleet run\n", *n, *clusters, min(max(*windows, 0), len(winServers)), *out, *out) if *demo { writeDemo(*out, *n, *windows) fmt.Printf("the demo: %s/demo.yaml has the lines for obs-beacond.yaml; run fleet with -demo and the same -n\n", *out) } } func hostName(i int) string { return fmt.Sprintf("fleet-%04d", i) } type stats struct { mu sync.Mutex lat []time.Duration errs map[string]int } func (s *stats) add(d time.Duration, err string) { s.mu.Lock() defer s.mu.Unlock() if err != "" { s.errs[err]++ return } s.lat = append(s.lat, d) } // take returns what came in since the last take, and starts again. func (s *stats) take() ([]time.Duration, map[string]int) { s.mu.Lock() defer s.mu.Unlock() l, e := s.lat, s.errs s.lat, s.errs = nil, map[string]int{} return l, e } func newStats() *stats { return &stats{errs: map[string]int{}} } func pct(l []time.Duration, p float64) time.Duration { if len(l) == 0 { return 0 } return l[min(len(l)-1, int(float64(len(l))*p))] } func summary(name string, l []time.Duration, e map[string]int, over time.Duration) string { slices.Sort(l) nerr := 0 for _, n := range e { nerr += n } s := fmt.Sprintf("%-8s %6d ok (%6.1f/s) p50 %7s p95 %7s p99 %7s max %7s errors %d", name, len(l), float64(len(l))/over.Seconds(), pct(l, .5).Round(100*time.Microsecond), pct(l, .95).Round(100*time.Microsecond), pct(l, .99).Round(100*time.Microsecond), pct(l, 1).Round(100*time.Microsecond), nerr) for k, n := range e { s += fmt.Sprintf("\n %5d × %s", n, k) } return s } func run(args []string) { fs := flag.NewFlagSet("run", flag.ExitOnError) servers := fs.String("server", "http://127.0.0.1:8150", "obs-beacond's address; several, comma-separated, are taken in turn as a load balancer would") tokFile := fs.String("tokens", "tokens", "from fleet config") n := fs.Int("n", 0, "agents to run (0: every token)") interval := fs.Duration("interval", 15*time.Second, "between one agent's reports") containers := fs.Int("containers", 10, "containers per host (the cluster watcher's pods come later)") withMap := fs.Bool("map", true, "send map samples: sites of 50 hosts, WireGuard between sites, apps talking to databases") dur := fs.Duration("for", 5*time.Minute, "how long") every := fs.Duration("print", 30*time.Second, "how often to print") ui := fs.Duration("ui", 2*time.Second, "between interface requests (0: none); they sign in with dev_signin") var shape clusterShape fs.IntVar(&shape.nodes, "nodes", 30, "nodes per cluster") fs.IntVar(&shape.controllers, "controllers", 300, "controllers per cluster") fs.IntVar(&shape.pods, "pods", 3000, "pods per cluster") fs.IntVar(&shape.vms, "vms", 300, "VMs per cluster, each with a volume") fs.Float64Var(&shape.churn, "churn", 2, "pod changes a second, per cluster") fs.DurationVar(&shape.relist, "relist", 30*time.Minute, "between full listings") fs.DurationVar(&shape.nodeDown, "node-down", 0, "after this long, cluster-00's node-01 goes down (0: never)") fs.DurationVar(&shape.nodeDownFor, "node-down-for", time.Minute, "how long node-01 stays down") fs.BoolVar(&shape.openshift, "openshift", false, "make cluster-01 OpenShift, mid-update, with one operator degraded") pveGuests := fs.Int("proxmox", 0, "this many VMs and containers in a Proxmox cluster read by fleet-0060's agent (0: none)") vsphereVMs := fs.Int("vsphere", 0, "this many VMs in a vCenter read by fleet-0080's agent (0: none)") ntnxVMs := fs.Int("nutanix", 0, "this many VMs in a Prism Central read by fleet-0100's agent (0: none)") pureN := fs.Int("pure", 0, "this many Pure arrays (up to 5: four FlashArrays with 600 volumes each, then a FlashBlade), all read by fleet-0040's agent (0: none)") ibmiN := fs.Int("ibmi", 0, "this many IBM i partitions (up to 4), and the HMC of their Power servers, all read by fleet-0041's agent (0: none)") akamaiDir := fs.String("akamai", "", "serve a fake Akamai account on loopback, and write its .edgerc and the akamai: section for obs-beacond.yaml to this directory's akamai.yaml (\"\": none)") chanDir := fs.String("channels", "", "catch alerts on loopback as ntfy, Discord, PagerDuty, a webhook and an email relay, and write their secrets and the alerts: and reports: sections for obs-beacond.yaml to this directory's channels.yaml (\"\": none)") mailDir := fs.String("mail", "", "serve a fake mail server on loopback, and write its password and the mail: section for obs-beacond.yaml to this directory's mail.yaml (\"\": none)") cfDir := fs.String("cloudflare", "", "serve a fake Cloudflare account on loopback, and write its token and the cloudflare: section for obs-beacond.yaml to this directory's cloudflare.yaml (\"\": none)") vaultDir := fs.String("vault", "", "serve two fake Vault clusters over HTTPS on loopback, and write their vault: list for obs-beacond.yaml to this directory's vault.yaml (\"\": none)") demo := fs.Bool("demo", false, "hosts report the demo's checks, units, backups and data layer, and fleet serves its Patroni cluster (fleet config -demo, same number of agents)") probe := fs.Bool("probe", true, "time how long a change in cluster-00 takes to show on every copy (needs dev_signin)") _ = fs.Parse(args) type agent struct{ host, token string } var agents, clusterAgents, winAgents []agent f, err := os.Open(*tokFile) if err != nil { log.Fatal(err) } sc := bufio.NewScanner(f) for sc.Scan() { h, t, ok := strings.Cut(sc.Text(), " ") switch { case ok && strings.HasPrefix(h, "cluster-"): clusterAgents = append(clusterAgents, agent{h, t}) case ok && strings.HasPrefix(h, winPrefix): if _, known := winServerFor(h); known { winAgents = append(winAgents, agent{h, t}) } case ok: agents = append(agents, agent{h, t}) } } f.Close() if *n > 0 && *n < len(agents) { agents = agents[:*n] } urls := strings.Split(*servers, ",") for _, u := range urls { if pu, err := url.Parse(u); err != nil || !isLoopback(pu.Hostname()) { log.Fatalf("fleet: %s: only a loopback test server", u) } } ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) defer stop() ctx, cancel := context.WithTimeout(ctx, *dur) defer cancel() client := &http.Client{Timeout: 30 * time.Second, Transport: &http.Transport{MaxIdleConnsPerHost: 256}} reports, pages, batches, shown := newStats(), newStats(), newStats(), newStats() // A load balancer: requests go only to copies whose /readyz passes, // checked every half second. ready := make([]atomic.Bool, len(urls)) for i := range ready { ready[i].Store(true) } go func() { hc := &http.Client{Timeout: time.Second} t := time.NewTicker(500 * time.Millisecond) defer t.Stop() for { for i, u := range urls { resp, err := hc.Get(u + "/readyz") ok := err == nil && resp.StatusCode == 200 if err == nil { resp.Body.Close() } if was := ready[i].Swap(ok); was != ok { log.Printf("%s lb: %s %s", time.Now().Format("15:04:05.000"), u, map[bool]string{true: "ready", false: "out"}[ok]) } } select { case <-ctx.Done(): return case <-t.C: } } }() var turn atomic.Uint64 pick := func() string { for range len(urls) { i := turn.Add(1) % uint64(len(urls)) if ready[i].Load() { return urls[i] } } return urls[turn.Load()%uint64(len(urls))] // none ready: try anyway, and count the failure } fleetSites = max(1, (len(agents)+siteSize-1)/siteSize) fleetClusters = len(clusterAgents) var demoAll []demoCheck if *demo { demoAll = demoChecks(len(agents)) if hasDataLayer(len(agents)) { servePatroni(ctx) } } var made, delivered, queued atomic.Int64 log.Printf("fleet: %d agents every %s, %d containers each, to %s, for %s", len(agents), *interval, *containers, *servers, *dur) var wg sync.WaitGroup for i, a := range append(slices.Clone(agents), winAgents...) { wg.Add(1) go func() { defer wg.Done() var h *host switch { case i >= len(agents): // a Windows server (windows.go) h = newWindowsHost(a.host, *interval) default: h = newHost(i, a.host, *containers, *interval) } if *demo && h.win == nil { h.demo = newDemoHost(fleetIndex(i, a.host), len(agents), demoAll) h.demo.setup(&h.r) } if *withMap && h.win == nil { h.m = fakeMap(i, a.host, h.r.Containers) h.certs, h.files = hostCerts(i, a.host) } // Agents start at random points in the interval, as a fleet does. select { case <-ctx.Done(): return case <-time.After(mrand.N(*interval)): } t := time.NewTicker(*interval) defer t.Stop() // Like the agent, keep what couldn't be sent and send it again, // oldest first, all but the newest marked as queued. var backlog [][]byte defer func() { queued.Add(int64(len(backlog))) }() send := func(body []byte, old bool) bool { start := time.Now() req, _ := http.NewRequestWithContext(ctx, "POST", pick()+"/api/v1/agent/report", bytes.NewReader(body)) req.Header.Set("Authorization", "Bearer "+a.token) req.Header.Set("Content-Type", "application/json") if old { req.Header.Set("X-Beacon-Queued", "1") } resp, err := client.Do(req) switch { case ctx.Err() != nil: return false case err != nil: reports.add(0, shortErr(err)) return false } _, _ = io.Copy(io.Discard, resp.Body) resp.Body.Close() if resp.StatusCode/100 != 2 { reports.add(0, fmt.Sprintf("HTTP %d", resp.StatusCode)) return false } reports.add(time.Since(start), "") delivered.Add(1) return true } for { body, _ := json.Marshal(h.next(time.Now())) made.Add(1) backlog = append(backlog, body) for len(backlog) > 0 && send(backlog[0], len(backlog) > 1) { backlog = backlog[1:] } select { case <-ctx.Done(): return case <-t.C: } } }() } if len(clusterAgents) > 0 { log.Printf("fleet: %d clusters, each %d nodes, %d controllers, %d pods, %d VMs; %.1f pod changes/s each", len(clusterAgents), shape.nodes, shape.controllers, shape.pods, shape.vms, shape.churn) } var cookies map[string]string if *probe && len(clusterAgents) > 0 { if cookies, err = signIn(ctx, client, urls); err != nil { log.Fatalf("fleet: %v", err) } } for _, a := range clusterAgents { wg.Add(1) go func() { defer wg.Done() c := newCluster(a.host, a.token, shape) var probed func(string, time.Time) if cookies != nil && a.host == "cluster-00" { probed = func(word string, at time.Time) { watchProbe(ctx, client, urls, cookies, word, at, shown) } } runCluster(ctx, c, client, pick, batches, probed) }() } if *pveGuests > 0 && len(agents) > proxmoxHost { a := agents[proxmoxHost] wg.Add(1) go func() { defer wg.Done() runCluster(ctx, newProxmox(a.host, a.token, *pveGuests), client, pick, batches, nil) }() log.Printf("fleet: Proxmox cluster %s, %d nodes, %d guests, read by %s", proxmoxCluster, proxmoxNodes, *pveGuests, a.host) } if *vsphereVMs > 0 && len(agents) > vcenterHost { a := agents[vcenterHost] wg.Add(1) go func() { defer wg.Done() runCluster(ctx, newVSphere(a.host, a.token, *vsphereVMs), client, pick, batches, nil) }() log.Printf("fleet: vCenter %s, %d ESXi hosts, %d VMs, read by %s", vcenterName, esxiHosts, *vsphereVMs, a.host) } if *vaultDir != "" { serveVaults(ctx, *vaultDir) } if *cfDir != "" { serveCloudflare(ctx, *cfDir) } if *mailDir != "" { serveMail(ctx, *mailDir) } if *chanDir != "" { serveChannels(ctx, *chanDir) } if *akamaiDir != "" { serveAkamai(ctx, *akamaiDir) } if *ntnxVMs > 0 && len(agents) > nutanixHost { a := agents[nutanixHost] wg.Add(1) go func() { defer wg.Done() runCluster(ctx, newNutanix(a.host, a.token, *ntnxVMs), client, pick, batches, nil) }() log.Printf("fleet: Prism Central %s, %d clusters of %d AHV hosts, %d VMs, read by %s", prismCentral, ntnxClusters, ahvPerPE, *ntnxVMs, a.host) } if n := min(*pureN, len(pureArrays)); n > 0 && len(agents) > pureHost { a := agents[pureHost] for k := range n { wg.Add(1) go func() { defer wg.Done() runCluster(ctx, newPure(a.host, a.token, k), client, pick, batches, nil) }() } log.Printf("fleet: %d Pure arrays (%v), read by %s", n, pureArrays[:n], a.host) } if n := min(*ibmiN, len(ibmiPartitions)); n > 0 && len(agents) > ibmiHost { a := agents[ibmiHost] for k := range n { wg.Add(1) go func() { defer wg.Done() runCluster(ctx, newIBMi(a.host, a.token, k), client, pick, batches, nil) }() } wg.Add(1) go func() { defer wg.Done() runCluster(ctx, newHMC(a.host, a.token), client, pick, batches, nil) }() log.Printf("fleet: %d IBM i partitions (%v) and the HMC %s, read by %s", n, ibmiPartitions[:n], hmcName, a.host) } if *ui > 0 && len(agents) > 0 { wg.Add(1) go func() { defer wg.Done() extra := []string{} if len(clusterAgents) > 0 { extra = []string{"/api/v1/clusters", "/api/v1/workloads?show=problems", "/api/v1/workloads?source=cluster-01&ns=ns-07&offset=100"} } browse(ctx, client, urls, pick, agents[0].host, *ui, pages, extra) }() } t := time.NewTicker(*every) defer t.Stop() last := time.Now() var allR, allP, allB, allS []time.Duration errR, errP, errB, errS := map[string]int{}, map[string]int{}, map[string]int{}, map[string]int{} print := func() { now := time.Now() l, e := reports.take() pl, pe := pages.take() line := fmt.Sprintf("%s\n%s\n%s", now.Format("15:04:05"), summary("reports", l, e, now.Sub(last)), summary("pages", pl, pe, now.Sub(last))) if len(clusterAgents) > 0 { bl, be := batches.take() sl, se := shown.take() line += "\n" + summary("batches", bl, be, now.Sub(last)) + "\n" + summary("shown", sl, se, now.Sub(last)) allB, allS = append(allB, bl...), append(allS, sl...) for k, v := range be { errB[k] += v } for k, v := range se { errS[k] += v } } log.Print(line) allR, allP = append(allR, l...), append(allP, pl...) for k, v := range e { errR[k] += v } for k, v := range pe { errP[k] += v } last = now } start := time.Now() loop: for { select { case <-ctx.Done(): break loop case <-t.C: print() } } wg.Wait() print() log.Printf("all %s\n%s\n%s", time.Since(start).Round(time.Second), summary("reports", allR, errR, time.Since(start)), summary("pages", allP, errP, time.Since(start))) if len(clusterAgents) > 0 { log.Printf("%s\n%s", summary("batches", allB, errB, time.Since(start)), summary("shown", allS, errS, time.Since(start))) } // Every report made is either delivered or still queued at the end; // none dropped. An agent sends its queue once obs-beacond is back. log.Printf("made %d reports: %d delivered, %d still queued at the end, %d lost", made.Load(), delivered.Load(), queued.Load(), made.Load()-delivered.Load()-queued.Load()) } func isLoopback(h string) bool { return h == "localhost" || h == "::1" || strings.HasPrefix(h, "127.") } func shortErr(err error) string { s := err.Error() if i := strings.LastIndex(s, ": "); i >= 0 { s = s[i+2:] } return s } // browse is someone looking at the interface: the pages' API calls in // turn, through the load balancer, signed in once on each copy (sessions // are shared, but each copy's own cookie keeps this independent of that). func browse(ctx context.Context, client *http.Client, urls []string, pick func() string, host string, every time.Duration, s *stats, extra []string) { cookies, err := signIn(ctx, client, urls) if err != nil { log.Printf("fleet: %v", err) return } paths := append([]string{"/api/v1/overview", "/api/v1/containers", "/api/v1/incidents", "/api/v1/hosts/" + host, "/api/v1/checks", "/api/v1/reports"}, extra...) t := time.NewTicker(every) defer t.Stop() for i := 0; ; i++ { u := pick() req, _ := http.NewRequestWithContext(ctx, "GET", u+paths[i%len(paths)], nil) req.Header.Set("Cookie", cookies[u]) start := time.Now() resp, err := client.Do(req) switch { case ctx.Err() != nil: return case err != nil: s.add(0, shortErr(err)) default: _, _ = io.Copy(io.Discard, resp.Body) resp.Body.Close() if resp.StatusCode != 200 { s.add(0, fmt.Sprintf("%s HTTP %d", paths[i%len(paths)], resp.StatusCode)) } else { s.add(time.Since(start), "") } } select { case <-ctx.Done(): return case <-t.C: } } } // host makes one fake host's reports: steady numbers with some noise, a // few disks, and containers that mostly run. type host struct { r report.Report start time.Time m *report.Map // its map sample, sent once a minute as the agent does mapSent time.Time vms []report.Workload // its libvirt VMs, when it's a hypervisor demo *demoHost // what it reports for the demo (demo.go), when it runs win *winHost // what a Windows server reports (windows.go) certs *report.CertScan // what its ports serve (certs.go) files *report.CertFiles // what its files hold certsSent time.Time } // hypervisorEvery says which fake hosts run libvirt: host 1 in every 20, // with libvirtVMs VMs. Hosts 2, 6, 10, 14 and 18 in the same 20 are its // first five VMs, with their domain UUIDs as SMBIOS UUIDs. const ( hypervisorEvery = 20 libvirtVMs = 12 ) var vmRoles = [libvirtVMs]string{"db", "web", "cache", "build", "mq", "ldap", "backup", "monitor", "runner", "jump", "proxy", "spare"} // hypervisorVMs are host i's domains: all running, but its spare is shut // off, and on the third hypervisor one has crashed. func hypervisorVMs(i int, now time.Time) []report.Workload { out := make([]report.Workload, libvirtVMs) for k := range out { id := vmUUID("libvirt-"+hostName(i), k) r := report.Workload{Kind: report.KindVM, ID: id, Name: fmt.Sprintf("%s-%02d", vmRoles[k], i/hypervisorEvery), State: report.StateOK, Word: "Running", Since: now.Add(-time.Duration(3+k) * 24 * time.Hour).Truncate(time.Second), Facts: map[string]any{"firmware_uuid": id, "autostart": k != libvirtVMs-1, "hypervisor": "libvirt", "vcpus": 2 << (k % 3), "memory_gib": float64(int(4) << (k % 3))}} switch { case k == libvirtVMs-1: r.State, r.Word = report.StateStopped, "Shut off" case i/hypervisorEvery == 2 && k == 7: r.State, r.Word, r.Since = report.StateDown, "Crashed", now.Add(-47*time.Minute).Truncate(time.Second) } out[k] = r } return out } // vmUsage is what the running VMs use now: a steady share each, with noise. func vmUsage(vms []report.Workload) []report.Usage { var out []report.Usage for k, r := range vms { if r.State != report.StateOK { continue } cpus, _ := r.Facts["vcpus"].(int) gib, _ := r.Facts["memory_gib"].(float64) share := 0.15 + 0.06*float64(k) + 0.1*mrand.Float64() out = append(out, report.Usage{ID: r.ID, CPU: math.Round(float64(cpus)*share*1000) / 1000, CPUOf: float64(cpus), Mem: int64(gib * (0.4 + 0.04*float64(k)) * (1 << 30)), MemOf: int64(gib * (1 << 30))}) } return out } // distro is one operating system in the fleet's mix. type distro struct { os, kernel string rootDev, rootFS string data string // where containers live } var ( rhel9 = distro{"Red Hat Enterprise Linux 9.6 (Plow)", "5.14.0-570.52.1.el9_6.x86_64", "/dev/mapper/rhel-root", "xfs", "/var/lib/containers"} rhel8 = distro{"Red Hat Enterprise Linux 8.10 (Ootpa)", "4.18.0-553.81.1.el8_10.x86_64", "/dev/mapper/rhel-root", "xfs", "/var/lib/containers"} rhel10 = distro{"Red Hat Enterprise Linux 10.0 (Coughlan)", "6.12.0-55.39.1.el10_0.x86_64", "/dev/mapper/rhel-root", "xfs", "/var/lib/containers"} ubuntu = distro{"Ubuntu 24.04.3 LTS", "6.8.0-87-generic", "/dev/nvme0n1p2", "ext4", "/var/lib/docker"} debian = distro{"Debian GNU/Linux 13 (trixie)", "6.12.48+deb13-amd64", "/dev/nvme0n1p2", "ext4", "/var/lib/docker"} ) // distroFor is a Red Hat shop's mix, in every 20 hosts: 12 on RHEL 9, // 3 still on RHEL 8, 2 already on RHEL 10, 2 Ubuntu and 1 Debian. func distroFor(i int) distro { switch n := i % 20; { case n == 7 || n == 13 || n == 18: return rhel8 case n == 4 || n == 15: return rhel10 case n == 9 || n == 17: return ubuntu case n == 11: return debian } return rhel9 } // vmHostEvery says which fake hosts are VMs: every fourth is cluster-00's // VM number i/4, with that VM's firmware UUID as its SMBIOS UUID, so the // server links them (docs/cluster-watcher.md, Linking a VM to its agent). const vmHostEvery = 4 // vmUUID is a fake VM's firmware UUID: the same for a cluster and number // on every run. func vmUUID(cluster string, i int) string { h := fnv.New32a() h.Write([]byte(cluster)) return fmt.Sprintf("%08x-0000-4000-8000-%012x", h.Sum32(), i) } // pvName is a fake PVC's PersistentVolume, as Kubernetes names it: what // the array volume behind it is named after (docs/pure.md, step 3). func pvName(cluster string, i int) string { return "pvc-" + vmUUID("pv-"+cluster, i) } func newHost(i int, name string, containers int, interval time.Duration) *host { h := &host{start: time.Now().Add(-time.Duration(mrand.N(30*24)) * time.Hour)} d := distroFor(i) h.r = report.Report{ V: report.Version, Agent: "fleet", Host: name, Interval: int(interval / time.Second), Kernel: d.kernel, OS: d.os, CPUs: 4 << (i % 3), Mem: report.Mem{Total: 16 << 30, SwapTotal: 4 << 30, SwapFree: 4 << 30}, Disks: []report.Disk{ {Mount: "/", Device: d.rootDev, FS: d.rootFS, Size: 100 << 30}, {Mount: d.data, Device: "/dev/nvme1n1", FS: "xfs", Size: 500 << 30}, }, Sources: []string{}, // what it reads beside itself, as the agent says } switch i { case proxmoxHost: h.r.Sources = []string{"proxmox"} case vcenterHost: h.r.Sources = []string{vcenterName} case nutanixHost: h.r.Sources = []string{prismCentral} case pureHost: h.r.Sources = pureArrays // what its pure: section lists, read or not case ibmiHost: h.r.Sources = append(append([]string{}, ibmiPartitions...), hmcName) } list := make([]report.Container, containers) for c := range list { list[c] = report.Container{ID: fmt.Sprintf("%012x", mrand.Uint64()), Name: fmt.Sprintf("app-%02d", c), Image: fmt.Sprintf("registry.example.org/app-%02d:1.%d", c, c%7), Project: "stack", State: "running", Status: "Up 3 days (healthy)", Health: "healthy", Created: h.start.Unix()} } h.r.Containers = &report.Containers{List: list} switch { case i%vmHostEvery == 0: h.r.Machine = vmUUID("cluster-00", i/vmHostEvery) case i%hypervisorEvery == 1: h.vms = hypervisorVMs(i, time.Now()) h.r.CPUs, h.r.Mem.Total = 64, 512<<30 case i%vmHostEvery == 2: h.r.Machine = vmUUID("libvirt-"+hostName(i/hypervisorEvery*hypervisorEvery+1), (i%hypervisorEvery-2)/vmHostEvery) case i%proxmoxGuestEvery == 3: h.r.Machine = vmUUID("proxmox-"+proxmoxCluster, i/proxmoxGuestEvery) case i%vsphereGuestEvery == 7: h.r.Machine = vmUUID("vsphere-"+vcenterName, i/vsphereGuestEvery) if (i/vsphereGuestEvery)%2 == 1 { h.r.Machine = report.SwappedUUID(h.r.Machine) // older virtual hardware } case i%nutanixGuestEvery == 11: h.r.Machine = vmUUID("nutanix-"+prismCentral, i/nutanixGuestEvery) } return h } func (h *host) next(now time.Time) report.Report { r := h.r r.Time = now.UTC() r.Uptime = now.Sub(h.start).Seconds() r.CPU = 5 + 30*mrand.Float64() r.Load = [3]float64{r.CPU / 25, r.CPU / 30, r.CPU / 35} r.Mem.Available = uint64(float64(r.Mem.Total) * (0.3 + 0.4*mrand.Float64())) r.Net = report.Net{RxBps: 1e5 * mrand.Float64(), TxBps: 5e4 * mrand.Float64()} disks := slices.Clone(r.Disks) for i := range disks { disks[i].Used = disks[i].Size / 3 disks[i].Avail = disks[i].Size - disks[i].Used } r.Disks = disks if h.vms != nil { r.VMs = &report.VMs{List: h.vms, Usage: vmUsage(h.vms)} } h.stampCerts(&r, now) if h.demo != nil { h.demo.stamp(&r, h.start, now) } if h.win != nil { h.win.stamp(h, &r, now) } if h.m != nil && now.Sub(h.mapSent) >= netmap.SendEvery-time.Duration(r.Interval)*time.Second/2 { // Handshakes are a minute or so old each time, as a live link's are. for d := range h.m.WireGuard { for p := range h.m.WireGuard[d].Peers { h.m.WireGuard[d].Peers[p].Handshake = now.Add(-time.Duration(30+p%60) * time.Second).UTC() } } r.Map, h.mapSent = h.m, now } return r }