Files
cairnobs/scripts/fleet/cluster.go
T
jcoffey-dev cba12cb2f3
ci / check (pull_request) Successful in 4m58s
ci / postgres (pull_request) Successful in 9m49s
Demo: problems come round about on a schedule, not like a clock
Every demo problem started at the same minute of each cycle and lasted
the same time, so Recently resolved read like a timetable, and 30 days
of history held about 3,700 incidents: app-04's container exiting 19
times a day for a month (#57).

Each occurrence's start now moves by up to an eighth of its period, its
length is 70% to 130% of the plan's, and half the periods have none,
four in five at night in Phoenix. All of it is seeded by the spell and
the period, so the simulator's parts still agree and a restarted demo
carries on where the clock is. The month of history now holds about
1,400 incidents with a working day's shape, and in the Phoenix day
there's never an hour with nothing going on (tested over three days).

Callers that stepped through a spell by its period or worked out its end
from its length use the occurrences instead: last, between.

Closes #57.
2026-11-15 23:40:34 -07:00

615 lines
23 KiB
Go

package main
// Fake cluster agents (docs/cluster-watcher.md, How it will be tested):
// each one is a cluster of nodes, controllers with their pods, VMs with
// their volumes, sending a full listing at start and every -relist, and
// changes every 5 s: pods restarting, a rollout now and then, a node that
// goes down if asked.
import (
"bytes"
"context"
"encoding/json"
"fmt"
"hash/fnv"
"io"
"math"
mrand "math/rand/v2"
"net/http"
"reflect"
"slices"
"strconv"
"strings"
"sync"
"time"
"github.com/cairnobs/cairnobs/internal/report"
)
type clusterShape struct {
nodes, controllers, pods, vms int
churn float64 // pod changes a second
relist time.Duration // between full listings
nodeDown time.Duration // after start, take a node down (0: never)
nodeDownFor time.Duration // for this long
openshift bool // cluster-01 is OpenShift, mid-update, with one operator degraded
}
type fakeCluster struct {
name, token string
shape clusterShape
mu sync.Mutex
gen string
recs map[string]report.Workload
pods []string // ids, for picking
changed map[string]bool
removed []string
seq int
noUsage bool // its reader sends none (Nutanix: usage is a separate API, not read yet)
source string // what a host agent calls it (a vCenter's host name); "" for a cluster agent
usageFn func(now time.Time) []report.Usage // its own readings instead of CPU and memory (an array's performance)
episodes []episode // records that go wrong and come right (scenario.go)
}
// episode is a record that takes its bad form while its spell is on.
type episode struct {
id string
s spell
good report.Workload
bad func(w *report.Workload, since time.Time)
on, in bool // on: in its bad form; in: played at least once
}
// flap makes record id go wrong during s and come right after
// (scenario.go): bad changes a copy of the record as it is now, given
// when the spell began.
func (c *fakeCluster) flap(id string, s spell, bad func(w *report.Workload, since time.Time)) {
if _, ok := c.recs[id]; !ok {
panic("flap: no record " + id)
}
c.episodes = append(c.episodes, episode{id: id, s: s, good: cloneWorkload(c.recs[id]), bad: bad})
c.play(time.Now())
}
// flapSet flaps every record that bad has and good hasn't, or has
// otherwise: a reader's records made from a good and a bad snapshot. A
// record only bad has is there during the spell and gone after.
func (c *fakeCluster) flapSet(s spell, good, bad []report.Workload) {
have := map[string]report.Workload{}
for _, r := range good {
have[r.ID] = r
}
for _, b := range bad {
g, ok := have[b.ID]
if ok {
g.Since, b.Since = time.Time{}, time.Time{}
if reflect.DeepEqual(g, b) {
continue
}
}
b := cloneWorkload(b)
if !ok {
c.episodes = append(c.episodes, episode{id: b.ID, s: s, bad: func(w *report.Workload, since time.Time) { *w = cloneWorkload(b); w.Since = since }})
continue
}
c.flap(b.ID, s, func(w *report.Workload, since time.Time) { *w = cloneWorkload(b); w.Since = since })
}
c.play(time.Now())
}
// play puts each flapping record in the form its spell says, and marks
// the ones that changed.
func (c *fakeCluster) play(now time.Time) {
c.mu.Lock()
defer c.mu.Unlock()
for i := range c.episodes {
e := &c.episodes[i]
on := e.s.active(now)
if e.in && on == e.on {
continue
}
e.on, e.in = on, true
if e.good.ID == "" && !on { // there only during the spell
if _, ok := c.recs[e.id]; ok {
delete(c.recs, e.id)
delete(c.changed, e.id)
c.removed = append(c.removed, e.id)
}
continue
}
w := cloneWorkload(e.good)
if on {
e.bad(&w, e.s.began(now))
} else if o := e.s.last(now); !e.good.Since.IsZero() && now.Sub(o.start) < e.s.every {
w.Since = o.end // right again since the spell ended
}
c.recs[e.id] = w
c.changed[e.id] = true
}
}
// flapping says whether record id has an episode, so churn leaves it be.
func (c *fakeCluster) flapping(id string) bool {
for _, e := range c.episodes {
if e.id == id {
return true
}
}
return false
}
func cloneWorkload(w report.Workload) report.Workload {
if w.Facts != nil {
f := make(map[string]any, len(w.Facts))
for k, v := range w.Facts {
f[k] = v
}
w.Facts = f
}
return w
}
func newID() string {
return fmt.Sprintf("%08x-%04x-%012x", mrand.Uint32(), mrand.Uint32()&0xffff, mrand.Uint64()&0xffffffffffff)
}
func newCluster(name, token string, s clusterShape) *fakeCluster {
c := &fakeCluster{name: name, token: token, shape: s, recs: map[string]report.Workload{}, changed: map[string]bool{}}
now := time.Now()
put := func(w report.Workload) { c.recs[w.ID] = w }
// OpenShift's nodes run Red Hat Enterprise Linux CoreOS; a plain
// Kubernetes cluster's, a general-purpose distribution.
kubelet, nodeOS := "v1.34.2", "Ubuntu 24.04.3 LTS"
if s.openshift && name == "cluster-01" {
kubelet, nodeOS = "v1.33.5", "Red Hat Enterprise Linux CoreOS 9.6.20251023-0 (Plow)"
}
put(report.Workload{Kind: report.KindCluster, ID: "cluster", Name: name, State: report.StateOK, Word: "API answering", Facts: map[string]any{"version": kubelet}})
for i := range s.nodes {
put(report.Workload{Kind: report.KindNode, ID: newID(), Name: fmt.Sprintf("node-%02d", i), State: report.StateOK, Word: "Ready",
Since: now.Add(-48 * time.Hour), Facts: map[string]any{"kubelet": kubelet, "os": nodeOS, "cpus": 32, "memory_gib": 256}})
}
k := clusterIndex(name)
crasher := ""
for i := range s.controllers {
ns, owner := fmt.Sprintf("ns-%02d", i%30), fmt.Sprintf("deployment/app-%03d", i)
want := s.pods / max(s.controllers, 1)
put(report.Workload{Kind: report.KindController, ID: newID(), Name: owner, Namespace: ns, State: report.StateOK,
Word: fmt.Sprintf("%d of %d ready", want, want), Facts: map[string]any{"desired": want, "ready": want},
Labels: map[string]string{"app.kubernetes.io/part-of": appNames[i%30]}})
put(svcRecord(fmt.Sprintf("app-%03d", i), ns, clusterIP(s.openshift && name == "cluster-01", k, i), "", "8080", []string{owner}, want, now))
if i%50 == 7 { // a canary Service whose Deployment is scaled to nothing
put(svcRecord(fmt.Sprintf("app-%03d-canary", i), ns, clusterIP(s.openshift && name == "cluster-01", k, 4000+i), "", "8080", nil, 0, now))
}
for range want {
c.addPod(ns, owner, now)
}
if i == 97 && k == 0 { // a pod that crashes for a while after each deploy (scenario.go)
crasher = c.pods[len(c.pods)-1]
}
}
vmGroup := map[string][]string{}
for i := range s.vms {
node := fmt.Sprintf("node-%02d", mrand.N(max(s.nodes, 1)))
g := vmGroups[i%len(vmGroups)]
vmGroup[g.name] = append(vmGroup[g.name], fmt.Sprintf("vm/vm-%03d", i))
// Its guest is what the fake host with its UUID says it runs.
put(report.Workload{Kind: report.KindVM, ID: newID(), Name: fmt.Sprintf("vm-%03d", i), Namespace: "vms", Node: node, State: report.StateOK,
Word: "Running", Since: now.Add(-24 * time.Hour), Facts: map[string]any{"guest_os": guestOS(distroFor(i * vmHostEvery).os),
"firmware_uuid": vmUUID(name, i), "run_strategy": "Always", "agent_connected": true, "vcpus": 4, "memory_gib": 8},
Labels: map[string]string{"app.kubernetes.io/part-of": g.name}})
put(report.Workload{Kind: report.KindPVC, ID: newID(), Name: fmt.Sprintf("vm-%03d-disk", i), Namespace: "vms", State: report.StateOK,
Word: "Bound", Facts: map[string]any{"storage_class": "pure-block", "capacity_gib": 50, "volume": pvName(name, i)}})
}
ocp := s.openshift && name == "cluster-01"
for j, g := range vmGroups {
if vms := vmGroup[g.name]; len(vms) > 0 {
put(svcRecord(g.name, "vms", clusterIP(ocp, k, 3000+j), "", g.port, vms, len(vms), now))
}
}
// The way in: one Ingress (a Route on OpenShift) per namespace, to its
// first app; and a load balancer the fleet's hosts call.
for n := range min(30, s.controllers) {
app := appNames[n]
kind, class := "ingress/", "nginx"
if ocp {
kind, class = "route/", "openshift-router"
}
host := fmt.Sprintf("%s.apps.%s.example.com", app, name)
put(report.Workload{Kind: report.KindIngress, ID: newID(), Name: kind + app, Namespace: fmt.Sprintf("ns-%02d", n), State: report.StateOK,
Word: fmt.Sprintf("%s → app-%03d", host, n), Since: now.Add(-30 * 24 * time.Hour),
Facts: map[string]any{"hosts": host, "backends": fmt.Sprintf("app-%03d:8080", n), "tls": true, "class": class}})
}
clusterCerts(name, ocp, min(30, s.controllers), put)
if s.controllers > 0 {
put(svcRecord("gateway", "ns-00", clusterIP(ocp, k, 3999), lbAddr(k), "443", []string{"deployment/app-000"}, 3, now))
}
if ocp {
part := func(id, n, state, word string, facts map[string]any) {
put(report.Workload{Kind: report.KindCluster, ID: id, Name: n, State: state, Word: word, Since: now.Add(-20 * time.Minute), Facts: facts})
}
part("os-cv", "clusterversion/version", report.StateUpdating, "Updating to 4.20.4: Working towards 4.20.4: 512 of 873 done (58% complete)",
map[string]any{"version": "4.20.3", "desired": "4.20.4", "distribution": "OpenShift"})
part("os-mcp-master", "machineconfigpool/master", report.StateOK, "Up to date (3 nodes)", nil)
part("os-mcp-worker", "machineconfigpool/worker", report.StateUpdating, fmt.Sprintf("Updating: 12 of %d nodes done", s.nodes), nil)
for i, n := range []string{"authentication", "console", "dns", "etcd", "image-registry", "kube-apiserver", "machine-config", "network", "storage"} {
part(fmt.Sprintf("os-co-%d", i), "clusteroperator/"+n, report.StateOK, "Available", nil)
}
part("os-co-ingress", "clusteroperator/ingress", report.StateDegraded, `Degraded: IngressDegraded: The "default" ingress controller reports Degraded=True`, nil)
part("os-co-monitoring", "clusteroperator/monitoring", report.StateUpdating, "Progressing: Rolling out the stack.", nil)
}
if name == "cluster-00" { // the pod the probe changes
put(report.Workload{Kind: report.KindPod, ID: "probe", Name: "probe", Namespace: "fleet", Node: "node-00", State: report.StateOK, Word: "Running"})
}
if crasher != "" {
c.flap(crasher, spellCrashloop, func(p *report.Workload, since time.Time) {
p.State, p.Word, p.Since = report.StateCrashloop, "Restarting (CrashLoopBackOff)", since
p.Facts["restarts"], p.Facts["last_exit"], p.Facts["ready"] = 9, "Error", "0/1"
})
}
return c
}
// The fake clusters' apps, one per namespace, and the VM groups.
var appNames = []string{"storefront", "checkout", "catalog", "payments", "search", "accounts", "orders", "inventory", "shipping", "pricing",
"reviews", "recommendations", "notifications", "billing", "ledger", "auth", "profile", "media", "reporting", "analytics",
"ingest", "scheduler", "returns", "loyalty", "fraud", "tax", "quotes", "support", "warehouse", "partners"}
var vmGroups = []struct{ name, port string }{{"erp", "8443"}, {"warehouse-db", "5432"}, {"file-servers", "445"}, {"directory", "636"},
{"build-agents", "22"}, {"legacy-crm", "8080"}, {"print", "631"}, {"backup-proxy", "8400"}, {"jump-hosts", "22"},
{"message-queue", "5671"}, {"license-servers", "27000"}, {"reporting-db", "1521"}}
func clusterIndex(name string) int {
n, _ := strconv.Atoi(strings.TrimPrefix(name, "cluster-"))
return n
}
// clusterIP is Service i's address: OpenShift's service network is
// 172.30.0.0/16, kubeadm's 10.96.0.0/12.
func clusterIP(openshift bool, k, i int) string {
if openshift {
return fmt.Sprintf("172.30.%d.%d", i/250, 10+i%250)
}
return fmt.Sprintf("10.%d.%d.%d", 96+k%16, i/250, 10+i%250)
}
// lbAddr is cluster k's load balancer, which some fleet hosts call on :443.
func lbAddr(k int) string { return fmt.Sprintf("10.200.%d.10", k%250) }
func svcRecord(name, ns, ip, external, ports string, backends []string, ready int, now time.Time) report.Workload {
w := report.Workload{Kind: report.KindService, ID: newID(), Name: name, Namespace: ns, State: report.StateOK,
Since: now.Add(-30 * 24 * time.Hour), Facts: map[string]any{"type": "ClusterIP", "cluster_ips": ip, "ports": ports}}
if external != "" {
w.Facts["type"], w.Facts["external"] = "LoadBalancer", external
}
if len(backends) > 0 {
w.Facts["backends"] = strings.Join(backends, ",")
}
switch ready {
case 0:
w.State, w.Word = report.StateStopped, "No ready endpoints"
case 1:
w.Word = "1 ready endpoint"
default:
w.Word = fmt.Sprintf("%d ready endpoints", ready)
}
return w
}
func (c *fakeCluster) addPod(ns, owner string, now time.Time) string {
id := newID()
c.recs[id] = report.Workload{Kind: report.KindPod, ID: id, Name: fmt.Sprintf("%s-%05x", strings.TrimPrefix(owner, "deployment/"), mrand.N(1<<20)),
Namespace: ns, Owner: owner, Node: fmt.Sprintf("node-%02d", mrand.N(max(c.shape.nodes, 1))), State: report.StateOK, Word: "Running",
Since: now.Add(-time.Duration(mrand.N(72)) * time.Hour), Labels: map[string]string{"app.kubernetes.io/name": strings.TrimPrefix(owner, "deployment/")},
Facts: map[string]any{"restarts": 0, "ready": "1/1"}}
c.pods = append(c.pods, id)
return id
}
// churn changes n pods: most restart once, the odd one is replaced.
func (c *fakeCluster) churn(n int, now time.Time) {
c.mu.Lock()
defer c.mu.Unlock()
for range n {
if len(c.pods) == 0 {
return
}
i := mrand.N(len(c.pods))
id := c.pods[i]
p := c.recs[id]
if p.State == report.StateCrashloop || c.flapping(id) {
continue
}
if mrand.N(10) == 0 { // replaced: a new pod, the old one gone
delete(c.recs, id)
c.removed = append(c.removed, id)
c.pods = slices.Delete(c.pods, i, i+1)
nid := c.addPod(p.Namespace, p.Owner, now)
q := c.recs[nid]
q.State, q.Word, q.Since = report.StateStarting, "ContainerCreating", now
c.recs[nid] = q
c.changed[nid] = true
continue
}
r, _ := p.Facts["restarts"].(int)
p.Facts = map[string]any{"restarts": r + 1, "ready": "1/1", "last_exit": "OOMKilled"}
p.State, p.Word, p.Since = report.StateOK, "Running", now
c.recs[id] = p
c.changed[id] = true
}
// Pods started last round are running now.
for id, p := range c.recs {
if p.Kind == report.KindPod && p.State == report.StateStarting && now.Sub(p.Since) > 4*time.Second {
p.State, p.Word = report.StateOK, "Running"
c.recs[id] = p
c.changed[id] = true
}
}
}
func (c *fakeCluster) setNode(name, state, word string, now time.Time) {
c.mu.Lock()
defer c.mu.Unlock()
for id, r := range c.recs {
if r.Kind == report.KindNode && r.Name == name {
r.State, r.Word, r.Since = state, word, now
c.recs[id] = r
c.changed[id] = true
}
}
}
// probe sets the probe pod's word, and returns it.
func (c *fakeCluster) probe(now time.Time) string {
c.mu.Lock()
defer c.mu.Unlock()
c.seq++
p := c.recs["probe"]
p.Word, p.Since = fmt.Sprintf("Running (probe %d)", c.seq), now
c.recs["probe"] = p
c.changed["probe"] = true
return p.Word
}
// batches is what to send now: a full listing in pages, or the changes.
func (c *fakeCluster) batches(full bool) []report.Workloads {
out := c.listing(full)
for i := range out {
out[i].Source = c.source
}
return out
}
func (c *fakeCluster) listing(full bool) []report.Workloads {
c.mu.Lock()
defer c.mu.Unlock()
if !full {
b := report.Workloads{V: report.Version, Agent: "fleet", Generation: c.gen, Records: []report.Workload{}, Removed: c.removed}
for id := range c.changed {
if r, ok := c.recs[id]; ok {
b.Records = append(b.Records, r)
}
}
c.changed, c.removed = map[string]bool{}, nil
return []report.Workloads{b}
}
c.gen = newID()
all := make([]report.Workload, 0, len(c.recs))
for _, r := range c.recs {
all = append(all, r)
}
c.changed, c.removed = map[string]bool{}, nil
var out []report.Workloads
for len(all) > 0 {
n := min(len(all), 500)
out = append(out, report.Workloads{V: report.Version, Agent: "fleet", Generation: c.gen, Full: report.FullPage, Records: all[:n]})
all = all[n:]
}
if len(out) == 0 {
out = append(out, report.Workloads{V: report.Version, Agent: "fleet", Generation: c.gen})
}
out[0].Full = report.FullBegin
out = append(out, report.Workloads{V: report.Version, Agent: "fleet", Generation: c.gen, Full: report.FullEnd, Records: []report.Workload{}})
return out
}
func (c *fakeCluster) size() int {
c.mu.Lock()
defer c.mu.Unlock()
return len(c.recs)
}
// runCluster plays one cluster agent until ctx ends. A batch that fails is
// sent again before anything newer, as the agent does.
func runCluster(ctx context.Context, c *fakeCluster, client *http.Client, pick func() string, s *stats, probed func(word string, at time.Time)) {
start := time.Now()
var backlog []report.Workloads
send := func() {
for len(backlog) > 0 {
body, _ := json.Marshal(backlog[0])
t0 := time.Now()
req, _ := http.NewRequestWithContext(ctx, "POST", pick()+"/api/v1/agent/workloads", bytes.NewReader(body))
req.Header.Set("Authorization", "Bearer "+c.token)
req.Header.Set("Content-Type", "application/json")
resp, err := client.Do(req)
if ctx.Err() != nil {
return
}
if err != nil {
s.add(0, shortErr(err))
return
}
msg, _ := io.ReadAll(io.LimitReader(resp.Body, 300))
resp.Body.Close()
if resp.StatusCode/100 != 2 {
s.add(0, fmt.Sprintf("HTTP %d %s", resp.StatusCode, strings.TrimSpace(string(msg))))
return
}
s.add(time.Since(t0), "")
backlog = backlog[1:]
}
}
backlog = append(backlog, c.batches(true)...)
send()
lastList := time.Now()
var lastUsage time.Time
t := time.NewTicker(5 * time.Second)
defer t.Stop()
down := false
var probeWord string
var probeAt time.Time
for {
select {
case <-ctx.Done():
return
case now := <-t.C:
if c.shape.nodeDown > 0 && c.name == "cluster-00" {
el := now.Sub(start)
if !down && el >= c.shape.nodeDown && el < c.shape.nodeDown+c.shape.nodeDownFor {
down = true
c.setNode("node-01", report.StateDown, "Not ready (kubelet stopped posting)", now)
fmt.Printf("%s fleet: cluster-00 node-01 down\n", now.Format("15:04:05.000"))
} else if down && el >= c.shape.nodeDown+c.shape.nodeDownFor {
down = false
c.shape.nodeDown = 0
c.setNode("node-01", report.StateOK, "Ready", now)
fmt.Printf("%s fleet: cluster-00 node-01 back\n", now.Format("15:04:05.000"))
}
}
c.churn(int(c.shape.churn*5), now)
c.play(now)
if probed != nil {
probeWord, probeAt = c.probe(now), now
}
if c.shape.relist > 0 && now.Sub(lastList) >= c.shape.relist {
backlog = append(backlog, c.batches(true)...)
lastList = now
} else {
backlog = append(backlog, c.batches(false)...)
}
// A usage reading a minute, as metrics.k8s.io gives the agent.
if (!c.noUsage || c.usageFn != nil) && now.Sub(lastUsage) >= time.Minute {
b := &backlog[len(backlog)-1]
b.Usage, b.UsageAt = c.usage(now), now.Truncate(time.Second)
if c.usageFn != nil {
b.Usage = c.usageFn(now)
}
lastUsage = now
}
send()
if probed != nil && len(backlog) == 0 {
probed(probeWord, probeAt)
}
}
}
}
// watchProbe times how long a probe change takes to show on each copy.
func watchProbe(ctx context.Context, client *http.Client, urls []string, cookies map[string]string, word string, at time.Time, s *stats) {
for _, u := range urls {
go func() {
for ctx.Err() == nil && time.Since(at) < 30*time.Second {
req, _ := http.NewRequestWithContext(ctx, "GET", u+"/api/v1/workloads?source=cluster-00&ns=fleet&q=probe", nil)
req.Header.Set("Cookie", cookies[u])
resp, err := client.Do(req)
if err == nil {
b, _ := io.ReadAll(resp.Body)
resp.Body.Close()
if bytes.Contains(b, []byte(`"`+word+`"`)) {
s.add(time.Since(at), "")
return
}
}
time.Sleep(50 * time.Millisecond)
}
if ctx.Err() == nil {
s.add(0, "not shown within 30 s on "+u)
}
}()
}
}
// signIn gets a dev sign-in cookie from each copy.
func signIn(ctx context.Context, client *http.Client, urls []string) (map[string]string, error) {
cookies := map[string]string{}
for _, u := range urls {
req, _ := http.NewRequestWithContext(ctx, "POST", u+"/auth/dev", nil)
req.Header.Set("X-Beacon", "1")
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("dev sign-in on %s: %w (is dev_signin on?)", u, err)
}
resp.Body.Close()
for _, c := range resp.Cookies() {
cookies[u] = c.Name + "=" + c.Value
}
if cookies[u] == "" {
return nil, fmt.Errorf("dev sign-in on %s: HTTP %d, no cookie (is dev_signin on?)", u, resp.StatusCode)
}
}
return cookies, nil
}
// guestOS is what KubeVirt's guest agent says: the name without the
// release's code name, "Red Hat Enterprise Linux 9.6".
func guestOS(os string) string {
name, _, _ := strings.Cut(os, " (")
return name
}
// usage is a reading for every node, controller and VM: each settles
// around its own level (from its id), drifting over the day, with a few
// running near their memory limit so the pages have something to show.
func (c *fakeCluster) usage(now time.Time) []report.Usage {
c.mu.Lock()
defer c.mu.Unlock()
day := math.Sin(2 * math.Pi * float64(now.Hour()*60+now.Minute()) / 1440) // -1 … 1 over the day
var out []report.Usage
for _, r := range c.recs {
h := fnv.New32a()
h.Write([]byte(r.ID))
seed := float64(h.Sum32()%1000) / 1000 // 0 … 1, fixed per record
level := 0.2 + 0.5*seed + 0.1*day + 0.05*(mrand.Float64()-0.5)
level = min(max(level, 0.02), 0.98)
// Memory moves less over the day than CPU, and sits elsewhere.
mem := min(max(0.3+0.45*float64(h.Sum32()/1000%1000)/1000+0.03*day, 0.05), 0.9)
switch r.Kind {
case report.KindNode:
if r.State != report.StateOK {
continue
}
cpus, gib := 32.0, 256.0 // a Kubernetes node's; Proxmox and ESXi nodes say their own
if n, ok := r.Facts["cpus"].(int); ok {
cpus = float64(n)
}
if g, ok := r.Facts["memory_gib"].(float64); ok {
gib = g
}
out = append(out, report.Usage{ID: r.ID, CPU: math.Round(cpus*level*1000) / 1000, Mem: int64(gib * mem * (1 << 30)), CPUOf: cpus, MemOf: int64(gib * (1 << 30))})
case report.KindController:
pods, _ := r.Facts["desired"].(int)
pods = max(pods, 1)
u := report.Usage{ID: r.ID, CPU: math.Round(float64(pods)*0.5*level*1000) / 1000, Mem: int64(float64(pods) * mem * (1 << 30)), Pods: pods}
if h.Sum32()%2 == 0 { // half have limits: half a core and 1 GiB a pod
u.CPUOf, u.MemOf = float64(pods)*0.5, int64(pods)<<30
if h.Sum32()%37 == 0 { // close to being OOM-killed
u.Mem = int64(float64(u.MemOf) * (0.92 + 0.05*mrand.Float64()))
}
}
out = append(out, u)
case report.KindVM:
if r.State == report.StateStopped {
continue
}
cpus, gib := 4.0, 8.0 // a KubeVirt VM's; a Proxmox guest says its own
if n, ok := r.Facts["vcpus"].(int); ok {
cpus = float64(n)
}
if g, ok := r.Facts["memory_gib"].(float64); ok {
gib = g
}
out = append(out, report.Usage{ID: r.ID, CPU: math.Round(cpus*level*1000) / 1000, Mem: int64(gib * mem * (1 << 30)), CPUOf: cpus, MemOf: int64(gib * (1 << 30)), Pods: 1})
}
}
return out
}