Every demo problem started at the same minute of each cycle and lasted the same time, so Recently resolved read like a timetable, and 30 days of history held about 3,700 incidents: app-04's container exiting 19 times a day for a month (#57). Each occurrence's start now moves by up to an eighth of its period, its length is 70% to 130% of the plan's, and half the periods have none, four in five at night in Phoenix. All of it is seeded by the spell and the period, so the simulator's parts still agree and a restarted demo carries on where the clock is. The month of history now holds about 1,400 incidents with a working day's shape, and in the Phoenix day there's never an hour with nothing going on (tested over three days). Callers that stepped through a spell by its period or worked out its end from its length use the occurrences instead: last, between. Closes #57.
615 lines
23 KiB
Go
615 lines
23 KiB
Go
package main
|
|
|
|
// Fake cluster agents (docs/cluster-watcher.md, How it will be tested):
|
|
// each one is a cluster of nodes, controllers with their pods, VMs with
|
|
// their volumes, sending a full listing at start and every -relist, and
|
|
// changes every 5 s: pods restarting, a rollout now and then, a node that
|
|
// goes down if asked.
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"hash/fnv"
|
|
"io"
|
|
"math"
|
|
mrand "math/rand/v2"
|
|
"net/http"
|
|
"reflect"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/cairnobs/cairnobs/internal/report"
|
|
)
|
|
|
|
type clusterShape struct {
|
|
nodes, controllers, pods, vms int
|
|
churn float64 // pod changes a second
|
|
relist time.Duration // between full listings
|
|
nodeDown time.Duration // after start, take a node down (0: never)
|
|
nodeDownFor time.Duration // for this long
|
|
openshift bool // cluster-01 is OpenShift, mid-update, with one operator degraded
|
|
}
|
|
|
|
type fakeCluster struct {
|
|
name, token string
|
|
shape clusterShape
|
|
mu sync.Mutex
|
|
gen string
|
|
recs map[string]report.Workload
|
|
pods []string // ids, for picking
|
|
changed map[string]bool
|
|
removed []string
|
|
seq int
|
|
noUsage bool // its reader sends none (Nutanix: usage is a separate API, not read yet)
|
|
source string // what a host agent calls it (a vCenter's host name); "" for a cluster agent
|
|
usageFn func(now time.Time) []report.Usage // its own readings instead of CPU and memory (an array's performance)
|
|
episodes []episode // records that go wrong and come right (scenario.go)
|
|
}
|
|
|
|
// episode is a record that takes its bad form while its spell is on.
|
|
type episode struct {
|
|
id string
|
|
s spell
|
|
good report.Workload
|
|
bad func(w *report.Workload, since time.Time)
|
|
on, in bool // on: in its bad form; in: played at least once
|
|
}
|
|
|
|
// flap makes record id go wrong during s and come right after
|
|
// (scenario.go): bad changes a copy of the record as it is now, given
|
|
// when the spell began.
|
|
func (c *fakeCluster) flap(id string, s spell, bad func(w *report.Workload, since time.Time)) {
|
|
if _, ok := c.recs[id]; !ok {
|
|
panic("flap: no record " + id)
|
|
}
|
|
c.episodes = append(c.episodes, episode{id: id, s: s, good: cloneWorkload(c.recs[id]), bad: bad})
|
|
c.play(time.Now())
|
|
}
|
|
|
|
// flapSet flaps every record that bad has and good hasn't, or has
|
|
// otherwise: a reader's records made from a good and a bad snapshot. A
|
|
// record only bad has is there during the spell and gone after.
|
|
func (c *fakeCluster) flapSet(s spell, good, bad []report.Workload) {
|
|
have := map[string]report.Workload{}
|
|
for _, r := range good {
|
|
have[r.ID] = r
|
|
}
|
|
for _, b := range bad {
|
|
g, ok := have[b.ID]
|
|
if ok {
|
|
g.Since, b.Since = time.Time{}, time.Time{}
|
|
if reflect.DeepEqual(g, b) {
|
|
continue
|
|
}
|
|
}
|
|
b := cloneWorkload(b)
|
|
if !ok {
|
|
c.episodes = append(c.episodes, episode{id: b.ID, s: s, bad: func(w *report.Workload, since time.Time) { *w = cloneWorkload(b); w.Since = since }})
|
|
continue
|
|
}
|
|
c.flap(b.ID, s, func(w *report.Workload, since time.Time) { *w = cloneWorkload(b); w.Since = since })
|
|
}
|
|
c.play(time.Now())
|
|
}
|
|
|
|
// play puts each flapping record in the form its spell says, and marks
|
|
// the ones that changed.
|
|
func (c *fakeCluster) play(now time.Time) {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
for i := range c.episodes {
|
|
e := &c.episodes[i]
|
|
on := e.s.active(now)
|
|
if e.in && on == e.on {
|
|
continue
|
|
}
|
|
e.on, e.in = on, true
|
|
if e.good.ID == "" && !on { // there only during the spell
|
|
if _, ok := c.recs[e.id]; ok {
|
|
delete(c.recs, e.id)
|
|
delete(c.changed, e.id)
|
|
c.removed = append(c.removed, e.id)
|
|
}
|
|
continue
|
|
}
|
|
w := cloneWorkload(e.good)
|
|
if on {
|
|
e.bad(&w, e.s.began(now))
|
|
} else if o := e.s.last(now); !e.good.Since.IsZero() && now.Sub(o.start) < e.s.every {
|
|
w.Since = o.end // right again since the spell ended
|
|
}
|
|
c.recs[e.id] = w
|
|
c.changed[e.id] = true
|
|
}
|
|
}
|
|
|
|
// flapping says whether record id has an episode, so churn leaves it be.
|
|
func (c *fakeCluster) flapping(id string) bool {
|
|
for _, e := range c.episodes {
|
|
if e.id == id {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func cloneWorkload(w report.Workload) report.Workload {
|
|
if w.Facts != nil {
|
|
f := make(map[string]any, len(w.Facts))
|
|
for k, v := range w.Facts {
|
|
f[k] = v
|
|
}
|
|
w.Facts = f
|
|
}
|
|
return w
|
|
}
|
|
|
|
func newID() string {
|
|
return fmt.Sprintf("%08x-%04x-%012x", mrand.Uint32(), mrand.Uint32()&0xffff, mrand.Uint64()&0xffffffffffff)
|
|
}
|
|
|
|
func newCluster(name, token string, s clusterShape) *fakeCluster {
|
|
c := &fakeCluster{name: name, token: token, shape: s, recs: map[string]report.Workload{}, changed: map[string]bool{}}
|
|
now := time.Now()
|
|
put := func(w report.Workload) { c.recs[w.ID] = w }
|
|
// OpenShift's nodes run Red Hat Enterprise Linux CoreOS; a plain
|
|
// Kubernetes cluster's, a general-purpose distribution.
|
|
kubelet, nodeOS := "v1.34.2", "Ubuntu 24.04.3 LTS"
|
|
if s.openshift && name == "cluster-01" {
|
|
kubelet, nodeOS = "v1.33.5", "Red Hat Enterprise Linux CoreOS 9.6.20251023-0 (Plow)"
|
|
}
|
|
put(report.Workload{Kind: report.KindCluster, ID: "cluster", Name: name, State: report.StateOK, Word: "API answering", Facts: map[string]any{"version": kubelet}})
|
|
for i := range s.nodes {
|
|
put(report.Workload{Kind: report.KindNode, ID: newID(), Name: fmt.Sprintf("node-%02d", i), State: report.StateOK, Word: "Ready",
|
|
Since: now.Add(-48 * time.Hour), Facts: map[string]any{"kubelet": kubelet, "os": nodeOS, "cpus": 32, "memory_gib": 256}})
|
|
}
|
|
k := clusterIndex(name)
|
|
crasher := ""
|
|
for i := range s.controllers {
|
|
ns, owner := fmt.Sprintf("ns-%02d", i%30), fmt.Sprintf("deployment/app-%03d", i)
|
|
want := s.pods / max(s.controllers, 1)
|
|
put(report.Workload{Kind: report.KindController, ID: newID(), Name: owner, Namespace: ns, State: report.StateOK,
|
|
Word: fmt.Sprintf("%d of %d ready", want, want), Facts: map[string]any{"desired": want, "ready": want},
|
|
Labels: map[string]string{"app.kubernetes.io/part-of": appNames[i%30]}})
|
|
put(svcRecord(fmt.Sprintf("app-%03d", i), ns, clusterIP(s.openshift && name == "cluster-01", k, i), "", "8080", []string{owner}, want, now))
|
|
if i%50 == 7 { // a canary Service whose Deployment is scaled to nothing
|
|
put(svcRecord(fmt.Sprintf("app-%03d-canary", i), ns, clusterIP(s.openshift && name == "cluster-01", k, 4000+i), "", "8080", nil, 0, now))
|
|
}
|
|
for range want {
|
|
c.addPod(ns, owner, now)
|
|
}
|
|
if i == 97 && k == 0 { // a pod that crashes for a while after each deploy (scenario.go)
|
|
crasher = c.pods[len(c.pods)-1]
|
|
}
|
|
}
|
|
vmGroup := map[string][]string{}
|
|
for i := range s.vms {
|
|
node := fmt.Sprintf("node-%02d", mrand.N(max(s.nodes, 1)))
|
|
g := vmGroups[i%len(vmGroups)]
|
|
vmGroup[g.name] = append(vmGroup[g.name], fmt.Sprintf("vm/vm-%03d", i))
|
|
// Its guest is what the fake host with its UUID says it runs.
|
|
put(report.Workload{Kind: report.KindVM, ID: newID(), Name: fmt.Sprintf("vm-%03d", i), Namespace: "vms", Node: node, State: report.StateOK,
|
|
Word: "Running", Since: now.Add(-24 * time.Hour), Facts: map[string]any{"guest_os": guestOS(distroFor(i * vmHostEvery).os),
|
|
"firmware_uuid": vmUUID(name, i), "run_strategy": "Always", "agent_connected": true, "vcpus": 4, "memory_gib": 8},
|
|
Labels: map[string]string{"app.kubernetes.io/part-of": g.name}})
|
|
put(report.Workload{Kind: report.KindPVC, ID: newID(), Name: fmt.Sprintf("vm-%03d-disk", i), Namespace: "vms", State: report.StateOK,
|
|
Word: "Bound", Facts: map[string]any{"storage_class": "pure-block", "capacity_gib": 50, "volume": pvName(name, i)}})
|
|
}
|
|
ocp := s.openshift && name == "cluster-01"
|
|
for j, g := range vmGroups {
|
|
if vms := vmGroup[g.name]; len(vms) > 0 {
|
|
put(svcRecord(g.name, "vms", clusterIP(ocp, k, 3000+j), "", g.port, vms, len(vms), now))
|
|
}
|
|
}
|
|
// The way in: one Ingress (a Route on OpenShift) per namespace, to its
|
|
// first app; and a load balancer the fleet's hosts call.
|
|
for n := range min(30, s.controllers) {
|
|
app := appNames[n]
|
|
kind, class := "ingress/", "nginx"
|
|
if ocp {
|
|
kind, class = "route/", "openshift-router"
|
|
}
|
|
host := fmt.Sprintf("%s.apps.%s.example.com", app, name)
|
|
put(report.Workload{Kind: report.KindIngress, ID: newID(), Name: kind + app, Namespace: fmt.Sprintf("ns-%02d", n), State: report.StateOK,
|
|
Word: fmt.Sprintf("%s → app-%03d", host, n), Since: now.Add(-30 * 24 * time.Hour),
|
|
Facts: map[string]any{"hosts": host, "backends": fmt.Sprintf("app-%03d:8080", n), "tls": true, "class": class}})
|
|
}
|
|
clusterCerts(name, ocp, min(30, s.controllers), put)
|
|
if s.controllers > 0 {
|
|
put(svcRecord("gateway", "ns-00", clusterIP(ocp, k, 3999), lbAddr(k), "443", []string{"deployment/app-000"}, 3, now))
|
|
}
|
|
if ocp {
|
|
part := func(id, n, state, word string, facts map[string]any) {
|
|
put(report.Workload{Kind: report.KindCluster, ID: id, Name: n, State: state, Word: word, Since: now.Add(-20 * time.Minute), Facts: facts})
|
|
}
|
|
part("os-cv", "clusterversion/version", report.StateUpdating, "Updating to 4.20.4: Working towards 4.20.4: 512 of 873 done (58% complete)",
|
|
map[string]any{"version": "4.20.3", "desired": "4.20.4", "distribution": "OpenShift"})
|
|
part("os-mcp-master", "machineconfigpool/master", report.StateOK, "Up to date (3 nodes)", nil)
|
|
part("os-mcp-worker", "machineconfigpool/worker", report.StateUpdating, fmt.Sprintf("Updating: 12 of %d nodes done", s.nodes), nil)
|
|
for i, n := range []string{"authentication", "console", "dns", "etcd", "image-registry", "kube-apiserver", "machine-config", "network", "storage"} {
|
|
part(fmt.Sprintf("os-co-%d", i), "clusteroperator/"+n, report.StateOK, "Available", nil)
|
|
}
|
|
part("os-co-ingress", "clusteroperator/ingress", report.StateDegraded, `Degraded: IngressDegraded: The "default" ingress controller reports Degraded=True`, nil)
|
|
part("os-co-monitoring", "clusteroperator/monitoring", report.StateUpdating, "Progressing: Rolling out the stack.", nil)
|
|
}
|
|
if name == "cluster-00" { // the pod the probe changes
|
|
put(report.Workload{Kind: report.KindPod, ID: "probe", Name: "probe", Namespace: "fleet", Node: "node-00", State: report.StateOK, Word: "Running"})
|
|
}
|
|
if crasher != "" {
|
|
c.flap(crasher, spellCrashloop, func(p *report.Workload, since time.Time) {
|
|
p.State, p.Word, p.Since = report.StateCrashloop, "Restarting (CrashLoopBackOff)", since
|
|
p.Facts["restarts"], p.Facts["last_exit"], p.Facts["ready"] = 9, "Error", "0/1"
|
|
})
|
|
}
|
|
return c
|
|
}
|
|
|
|
// The fake clusters' apps, one per namespace, and the VM groups.
|
|
var appNames = []string{"storefront", "checkout", "catalog", "payments", "search", "accounts", "orders", "inventory", "shipping", "pricing",
|
|
"reviews", "recommendations", "notifications", "billing", "ledger", "auth", "profile", "media", "reporting", "analytics",
|
|
"ingest", "scheduler", "returns", "loyalty", "fraud", "tax", "quotes", "support", "warehouse", "partners"}
|
|
|
|
var vmGroups = []struct{ name, port string }{{"erp", "8443"}, {"warehouse-db", "5432"}, {"file-servers", "445"}, {"directory", "636"},
|
|
{"build-agents", "22"}, {"legacy-crm", "8080"}, {"print", "631"}, {"backup-proxy", "8400"}, {"jump-hosts", "22"},
|
|
{"message-queue", "5671"}, {"license-servers", "27000"}, {"reporting-db", "1521"}}
|
|
|
|
func clusterIndex(name string) int {
|
|
n, _ := strconv.Atoi(strings.TrimPrefix(name, "cluster-"))
|
|
return n
|
|
}
|
|
|
|
// clusterIP is Service i's address: OpenShift's service network is
|
|
// 172.30.0.0/16, kubeadm's 10.96.0.0/12.
|
|
func clusterIP(openshift bool, k, i int) string {
|
|
if openshift {
|
|
return fmt.Sprintf("172.30.%d.%d", i/250, 10+i%250)
|
|
}
|
|
return fmt.Sprintf("10.%d.%d.%d", 96+k%16, i/250, 10+i%250)
|
|
}
|
|
|
|
// lbAddr is cluster k's load balancer, which some fleet hosts call on :443.
|
|
func lbAddr(k int) string { return fmt.Sprintf("10.200.%d.10", k%250) }
|
|
|
|
func svcRecord(name, ns, ip, external, ports string, backends []string, ready int, now time.Time) report.Workload {
|
|
w := report.Workload{Kind: report.KindService, ID: newID(), Name: name, Namespace: ns, State: report.StateOK,
|
|
Since: now.Add(-30 * 24 * time.Hour), Facts: map[string]any{"type": "ClusterIP", "cluster_ips": ip, "ports": ports}}
|
|
if external != "" {
|
|
w.Facts["type"], w.Facts["external"] = "LoadBalancer", external
|
|
}
|
|
if len(backends) > 0 {
|
|
w.Facts["backends"] = strings.Join(backends, ",")
|
|
}
|
|
switch ready {
|
|
case 0:
|
|
w.State, w.Word = report.StateStopped, "No ready endpoints"
|
|
case 1:
|
|
w.Word = "1 ready endpoint"
|
|
default:
|
|
w.Word = fmt.Sprintf("%d ready endpoints", ready)
|
|
}
|
|
return w
|
|
}
|
|
|
|
func (c *fakeCluster) addPod(ns, owner string, now time.Time) string {
|
|
id := newID()
|
|
c.recs[id] = report.Workload{Kind: report.KindPod, ID: id, Name: fmt.Sprintf("%s-%05x", strings.TrimPrefix(owner, "deployment/"), mrand.N(1<<20)),
|
|
Namespace: ns, Owner: owner, Node: fmt.Sprintf("node-%02d", mrand.N(max(c.shape.nodes, 1))), State: report.StateOK, Word: "Running",
|
|
Since: now.Add(-time.Duration(mrand.N(72)) * time.Hour), Labels: map[string]string{"app.kubernetes.io/name": strings.TrimPrefix(owner, "deployment/")},
|
|
Facts: map[string]any{"restarts": 0, "ready": "1/1"}}
|
|
c.pods = append(c.pods, id)
|
|
return id
|
|
}
|
|
|
|
// churn changes n pods: most restart once, the odd one is replaced.
|
|
func (c *fakeCluster) churn(n int, now time.Time) {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
for range n {
|
|
if len(c.pods) == 0 {
|
|
return
|
|
}
|
|
i := mrand.N(len(c.pods))
|
|
id := c.pods[i]
|
|
p := c.recs[id]
|
|
if p.State == report.StateCrashloop || c.flapping(id) {
|
|
continue
|
|
}
|
|
if mrand.N(10) == 0 { // replaced: a new pod, the old one gone
|
|
delete(c.recs, id)
|
|
c.removed = append(c.removed, id)
|
|
c.pods = slices.Delete(c.pods, i, i+1)
|
|
nid := c.addPod(p.Namespace, p.Owner, now)
|
|
q := c.recs[nid]
|
|
q.State, q.Word, q.Since = report.StateStarting, "ContainerCreating", now
|
|
c.recs[nid] = q
|
|
c.changed[nid] = true
|
|
continue
|
|
}
|
|
r, _ := p.Facts["restarts"].(int)
|
|
p.Facts = map[string]any{"restarts": r + 1, "ready": "1/1", "last_exit": "OOMKilled"}
|
|
p.State, p.Word, p.Since = report.StateOK, "Running", now
|
|
c.recs[id] = p
|
|
c.changed[id] = true
|
|
}
|
|
// Pods started last round are running now.
|
|
for id, p := range c.recs {
|
|
if p.Kind == report.KindPod && p.State == report.StateStarting && now.Sub(p.Since) > 4*time.Second {
|
|
p.State, p.Word = report.StateOK, "Running"
|
|
c.recs[id] = p
|
|
c.changed[id] = true
|
|
}
|
|
}
|
|
}
|
|
|
|
func (c *fakeCluster) setNode(name, state, word string, now time.Time) {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
for id, r := range c.recs {
|
|
if r.Kind == report.KindNode && r.Name == name {
|
|
r.State, r.Word, r.Since = state, word, now
|
|
c.recs[id] = r
|
|
c.changed[id] = true
|
|
}
|
|
}
|
|
}
|
|
|
|
// probe sets the probe pod's word, and returns it.
|
|
func (c *fakeCluster) probe(now time.Time) string {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
c.seq++
|
|
p := c.recs["probe"]
|
|
p.Word, p.Since = fmt.Sprintf("Running (probe %d)", c.seq), now
|
|
c.recs["probe"] = p
|
|
c.changed["probe"] = true
|
|
return p.Word
|
|
}
|
|
|
|
// batches is what to send now: a full listing in pages, or the changes.
|
|
func (c *fakeCluster) batches(full bool) []report.Workloads {
|
|
out := c.listing(full)
|
|
for i := range out {
|
|
out[i].Source = c.source
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (c *fakeCluster) listing(full bool) []report.Workloads {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
if !full {
|
|
b := report.Workloads{V: report.Version, Agent: "fleet", Generation: c.gen, Records: []report.Workload{}, Removed: c.removed}
|
|
for id := range c.changed {
|
|
if r, ok := c.recs[id]; ok {
|
|
b.Records = append(b.Records, r)
|
|
}
|
|
}
|
|
c.changed, c.removed = map[string]bool{}, nil
|
|
return []report.Workloads{b}
|
|
}
|
|
c.gen = newID()
|
|
all := make([]report.Workload, 0, len(c.recs))
|
|
for _, r := range c.recs {
|
|
all = append(all, r)
|
|
}
|
|
c.changed, c.removed = map[string]bool{}, nil
|
|
var out []report.Workloads
|
|
for len(all) > 0 {
|
|
n := min(len(all), 500)
|
|
out = append(out, report.Workloads{V: report.Version, Agent: "fleet", Generation: c.gen, Full: report.FullPage, Records: all[:n]})
|
|
all = all[n:]
|
|
}
|
|
if len(out) == 0 {
|
|
out = append(out, report.Workloads{V: report.Version, Agent: "fleet", Generation: c.gen})
|
|
}
|
|
out[0].Full = report.FullBegin
|
|
out = append(out, report.Workloads{V: report.Version, Agent: "fleet", Generation: c.gen, Full: report.FullEnd, Records: []report.Workload{}})
|
|
return out
|
|
}
|
|
|
|
func (c *fakeCluster) size() int {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
return len(c.recs)
|
|
}
|
|
|
|
// runCluster plays one cluster agent until ctx ends. A batch that fails is
|
|
// sent again before anything newer, as the agent does.
|
|
func runCluster(ctx context.Context, c *fakeCluster, client *http.Client, pick func() string, s *stats, probed func(word string, at time.Time)) {
|
|
start := time.Now()
|
|
var backlog []report.Workloads
|
|
send := func() {
|
|
for len(backlog) > 0 {
|
|
body, _ := json.Marshal(backlog[0])
|
|
t0 := time.Now()
|
|
req, _ := http.NewRequestWithContext(ctx, "POST", pick()+"/api/v1/agent/workloads", bytes.NewReader(body))
|
|
req.Header.Set("Authorization", "Bearer "+c.token)
|
|
req.Header.Set("Content-Type", "application/json")
|
|
resp, err := client.Do(req)
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
if err != nil {
|
|
s.add(0, shortErr(err))
|
|
return
|
|
}
|
|
msg, _ := io.ReadAll(io.LimitReader(resp.Body, 300))
|
|
resp.Body.Close()
|
|
if resp.StatusCode/100 != 2 {
|
|
s.add(0, fmt.Sprintf("HTTP %d %s", resp.StatusCode, strings.TrimSpace(string(msg))))
|
|
return
|
|
}
|
|
s.add(time.Since(t0), "")
|
|
backlog = backlog[1:]
|
|
}
|
|
}
|
|
backlog = append(backlog, c.batches(true)...)
|
|
send()
|
|
lastList := time.Now()
|
|
var lastUsage time.Time
|
|
t := time.NewTicker(5 * time.Second)
|
|
defer t.Stop()
|
|
down := false
|
|
var probeWord string
|
|
var probeAt time.Time
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case now := <-t.C:
|
|
if c.shape.nodeDown > 0 && c.name == "cluster-00" {
|
|
el := now.Sub(start)
|
|
if !down && el >= c.shape.nodeDown && el < c.shape.nodeDown+c.shape.nodeDownFor {
|
|
down = true
|
|
c.setNode("node-01", report.StateDown, "Not ready (kubelet stopped posting)", now)
|
|
fmt.Printf("%s fleet: cluster-00 node-01 down\n", now.Format("15:04:05.000"))
|
|
} else if down && el >= c.shape.nodeDown+c.shape.nodeDownFor {
|
|
down = false
|
|
c.shape.nodeDown = 0
|
|
c.setNode("node-01", report.StateOK, "Ready", now)
|
|
fmt.Printf("%s fleet: cluster-00 node-01 back\n", now.Format("15:04:05.000"))
|
|
}
|
|
}
|
|
c.churn(int(c.shape.churn*5), now)
|
|
c.play(now)
|
|
if probed != nil {
|
|
probeWord, probeAt = c.probe(now), now
|
|
}
|
|
if c.shape.relist > 0 && now.Sub(lastList) >= c.shape.relist {
|
|
backlog = append(backlog, c.batches(true)...)
|
|
lastList = now
|
|
} else {
|
|
backlog = append(backlog, c.batches(false)...)
|
|
}
|
|
// A usage reading a minute, as metrics.k8s.io gives the agent.
|
|
if (!c.noUsage || c.usageFn != nil) && now.Sub(lastUsage) >= time.Minute {
|
|
b := &backlog[len(backlog)-1]
|
|
b.Usage, b.UsageAt = c.usage(now), now.Truncate(time.Second)
|
|
if c.usageFn != nil {
|
|
b.Usage = c.usageFn(now)
|
|
}
|
|
lastUsage = now
|
|
}
|
|
send()
|
|
if probed != nil && len(backlog) == 0 {
|
|
probed(probeWord, probeAt)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// watchProbe times how long a probe change takes to show on each copy.
|
|
func watchProbe(ctx context.Context, client *http.Client, urls []string, cookies map[string]string, word string, at time.Time, s *stats) {
|
|
for _, u := range urls {
|
|
go func() {
|
|
for ctx.Err() == nil && time.Since(at) < 30*time.Second {
|
|
req, _ := http.NewRequestWithContext(ctx, "GET", u+"/api/v1/workloads?source=cluster-00&ns=fleet&q=probe", nil)
|
|
req.Header.Set("Cookie", cookies[u])
|
|
resp, err := client.Do(req)
|
|
if err == nil {
|
|
b, _ := io.ReadAll(resp.Body)
|
|
resp.Body.Close()
|
|
if bytes.Contains(b, []byte(`"`+word+`"`)) {
|
|
s.add(time.Since(at), "")
|
|
return
|
|
}
|
|
}
|
|
time.Sleep(50 * time.Millisecond)
|
|
}
|
|
if ctx.Err() == nil {
|
|
s.add(0, "not shown within 30 s on "+u)
|
|
}
|
|
}()
|
|
}
|
|
}
|
|
|
|
// signIn gets a dev sign-in cookie from each copy.
|
|
func signIn(ctx context.Context, client *http.Client, urls []string) (map[string]string, error) {
|
|
cookies := map[string]string{}
|
|
for _, u := range urls {
|
|
req, _ := http.NewRequestWithContext(ctx, "POST", u+"/auth/dev", nil)
|
|
req.Header.Set("X-Beacon", "1")
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("dev sign-in on %s: %w (is dev_signin on?)", u, err)
|
|
}
|
|
resp.Body.Close()
|
|
for _, c := range resp.Cookies() {
|
|
cookies[u] = c.Name + "=" + c.Value
|
|
}
|
|
if cookies[u] == "" {
|
|
return nil, fmt.Errorf("dev sign-in on %s: HTTP %d, no cookie (is dev_signin on?)", u, resp.StatusCode)
|
|
}
|
|
}
|
|
return cookies, nil
|
|
}
|
|
|
|
// guestOS is what KubeVirt's guest agent says: the name without the
|
|
// release's code name, "Red Hat Enterprise Linux 9.6".
|
|
func guestOS(os string) string {
|
|
name, _, _ := strings.Cut(os, " (")
|
|
return name
|
|
}
|
|
|
|
// usage is a reading for every node, controller and VM: each settles
|
|
// around its own level (from its id), drifting over the day, with a few
|
|
// running near their memory limit so the pages have something to show.
|
|
func (c *fakeCluster) usage(now time.Time) []report.Usage {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
day := math.Sin(2 * math.Pi * float64(now.Hour()*60+now.Minute()) / 1440) // -1 … 1 over the day
|
|
var out []report.Usage
|
|
for _, r := range c.recs {
|
|
h := fnv.New32a()
|
|
h.Write([]byte(r.ID))
|
|
seed := float64(h.Sum32()%1000) / 1000 // 0 … 1, fixed per record
|
|
level := 0.2 + 0.5*seed + 0.1*day + 0.05*(mrand.Float64()-0.5)
|
|
level = min(max(level, 0.02), 0.98)
|
|
// Memory moves less over the day than CPU, and sits elsewhere.
|
|
mem := min(max(0.3+0.45*float64(h.Sum32()/1000%1000)/1000+0.03*day, 0.05), 0.9)
|
|
switch r.Kind {
|
|
case report.KindNode:
|
|
if r.State != report.StateOK {
|
|
continue
|
|
}
|
|
cpus, gib := 32.0, 256.0 // a Kubernetes node's; Proxmox and ESXi nodes say their own
|
|
if n, ok := r.Facts["cpus"].(int); ok {
|
|
cpus = float64(n)
|
|
}
|
|
if g, ok := r.Facts["memory_gib"].(float64); ok {
|
|
gib = g
|
|
}
|
|
out = append(out, report.Usage{ID: r.ID, CPU: math.Round(cpus*level*1000) / 1000, Mem: int64(gib * mem * (1 << 30)), CPUOf: cpus, MemOf: int64(gib * (1 << 30))})
|
|
case report.KindController:
|
|
pods, _ := r.Facts["desired"].(int)
|
|
pods = max(pods, 1)
|
|
u := report.Usage{ID: r.ID, CPU: math.Round(float64(pods)*0.5*level*1000) / 1000, Mem: int64(float64(pods) * mem * (1 << 30)), Pods: pods}
|
|
if h.Sum32()%2 == 0 { // half have limits: half a core and 1 GiB a pod
|
|
u.CPUOf, u.MemOf = float64(pods)*0.5, int64(pods)<<30
|
|
if h.Sum32()%37 == 0 { // close to being OOM-killed
|
|
u.Mem = int64(float64(u.MemOf) * (0.92 + 0.05*mrand.Float64()))
|
|
}
|
|
}
|
|
out = append(out, u)
|
|
case report.KindVM:
|
|
if r.State == report.StateStopped {
|
|
continue
|
|
}
|
|
cpus, gib := 4.0, 8.0 // a KubeVirt VM's; a Proxmox guest says its own
|
|
if n, ok := r.Facts["vcpus"].(int); ok {
|
|
cpus = float64(n)
|
|
}
|
|
if g, ok := r.Facts["memory_gib"].(float64); ok {
|
|
gib = g
|
|
}
|
|
out = append(out, report.Usage{ID: r.ID, CPU: math.Round(cpus*level*1000) / 1000, Mem: int64(gib * mem * (1 << 30)), CPUOf: cpus, MemOf: int64(gib * (1 << 30)), Pods: 1})
|
|
}
|
|
}
|
|
return out
|
|
}
|