Files
cairnobs/scripts/fleet/cluster.go
T
jcoffey-dev db156c4f06
release / release (push) Successful in 7m6s
Cairn OBS
The new Cairn OBS: a self-hosted monitor for networks, hosts and
services. Licensed AGPLv3.
2026-11-14 03:27:09 -07:00

510 lines
20 KiB
Go

package main
// Fake cluster agents (docs/cluster-watcher.md, How it will be tested):
// each one is a cluster of nodes, controllers with their pods, VMs with
// their volumes, sending a full listing at start and every -relist, and
// changes every 5 s: pods restarting, a rollout now and then, a node that
// goes down if asked.
import (
"bytes"
"context"
"encoding/json"
"fmt"
"hash/fnv"
"io"
"math"
mrand "math/rand/v2"
"net/http"
"slices"
"strconv"
"strings"
"sync"
"time"
"github.com/cairnobs/cairnobs/internal/report"
)
type clusterShape struct {
nodes, controllers, pods, vms int
churn float64 // pod changes a second
relist time.Duration // between full listings
nodeDown time.Duration // after start, take a node down (0: never)
nodeDownFor time.Duration // for this long
openshift bool // cluster-01 is OpenShift, mid-update, with one operator degraded
}
type fakeCluster struct {
name, token string
shape clusterShape
mu sync.Mutex
gen string
recs map[string]report.Workload
pods []string // ids, for picking
changed map[string]bool
removed []string
seq int
noUsage bool // its reader sends none (Nutanix: usage is a separate API, not read yet)
source string // what a host agent calls it (a vCenter's host name); "" for a cluster agent
usageFn func(now time.Time) []report.Usage // its own readings instead of CPU and memory (an array's performance)
}
func newID() string {
return fmt.Sprintf("%08x-%04x-%012x", mrand.Uint32(), mrand.Uint32()&0xffff, mrand.Uint64()&0xffffffffffff)
}
func newCluster(name, token string, s clusterShape) *fakeCluster {
c := &fakeCluster{name: name, token: token, shape: s, recs: map[string]report.Workload{}, changed: map[string]bool{}}
now := time.Now()
put := func(w report.Workload) { c.recs[w.ID] = w }
// OpenShift's nodes run Red Hat Enterprise Linux CoreOS; a plain
// Kubernetes cluster's, a general-purpose distribution.
kubelet, nodeOS := "v1.34.2", "Ubuntu 24.04.3 LTS"
if s.openshift && name == "cluster-01" {
kubelet, nodeOS = "v1.33.5", "Red Hat Enterprise Linux CoreOS 9.6.20251023-0 (Plow)"
}
put(report.Workload{Kind: report.KindCluster, ID: "cluster", Name: name, State: report.StateOK, Word: "API answering", Facts: map[string]any{"version": kubelet}})
for i := range s.nodes {
put(report.Workload{Kind: report.KindNode, ID: newID(), Name: fmt.Sprintf("node-%02d", i), State: report.StateOK, Word: "Ready",
Since: now.Add(-48 * time.Hour), Facts: map[string]any{"kubelet": kubelet, "os": nodeOS, "cpus": 32, "memory_gib": 256}})
}
k := clusterIndex(name)
for i := range s.controllers {
ns, owner := fmt.Sprintf("ns-%02d", i%30), fmt.Sprintf("deployment/app-%03d", i)
want := s.pods / max(s.controllers, 1)
put(report.Workload{Kind: report.KindController, ID: newID(), Name: owner, Namespace: ns, State: report.StateOK,
Word: fmt.Sprintf("%d of %d ready", want, want), Facts: map[string]any{"desired": want, "ready": want},
Labels: map[string]string{"app.kubernetes.io/part-of": appNames[i%30]}})
put(svcRecord(fmt.Sprintf("app-%03d", i), ns, clusterIP(s.openshift && name == "cluster-01", k, i), "", "8080", []string{owner}, want, now))
if i%50 == 7 { // a canary Service whose Deployment is scaled to nothing
put(svcRecord(fmt.Sprintf("app-%03d-canary", i), ns, clusterIP(s.openshift && name == "cluster-01", k, 4000+i), "", "8080", nil, 0, now))
}
for range want {
c.addPod(ns, owner, now)
}
if i%97 == 0 { // a few controllers with a crashing pod
id := c.pods[len(c.pods)-1]
p := c.recs[id]
p.State, p.Word, p.Facts = report.StateCrashloop, "Restarting (CrashLoopBackOff)", map[string]any{"restarts": 40, "last_exit": "Error"}
put(p)
}
}
vmGroup := map[string][]string{}
for i := range s.vms {
node := fmt.Sprintf("node-%02d", mrand.N(max(s.nodes, 1)))
g := vmGroups[i%len(vmGroups)]
vmGroup[g.name] = append(vmGroup[g.name], fmt.Sprintf("vm/vm-%03d", i))
// Its guest is what the fake host with its UUID says it runs.
put(report.Workload{Kind: report.KindVM, ID: newID(), Name: fmt.Sprintf("vm-%03d", i), Namespace: "vms", Node: node, State: report.StateOK,
Word: "Running", Since: now.Add(-24 * time.Hour), Facts: map[string]any{"guest_os": guestOS(distroFor(i * vmHostEvery).os),
"firmware_uuid": vmUUID(name, i), "run_strategy": "Always", "agent_connected": true, "vcpus": 4, "memory_gib": 8},
Labels: map[string]string{"app.kubernetes.io/part-of": g.name}})
put(report.Workload{Kind: report.KindPVC, ID: newID(), Name: fmt.Sprintf("vm-%03d-disk", i), Namespace: "vms", State: report.StateOK,
Word: "Bound", Facts: map[string]any{"storage_class": "pure-block", "capacity_gib": 50, "volume": pvName(name, i)}})
}
ocp := s.openshift && name == "cluster-01"
for j, g := range vmGroups {
if vms := vmGroup[g.name]; len(vms) > 0 {
put(svcRecord(g.name, "vms", clusterIP(ocp, k, 3000+j), "", g.port, vms, len(vms), now))
}
}
// The way in: one Ingress (a Route on OpenShift) per namespace, to its
// first app; and a load balancer the fleet's hosts call.
for n := range min(30, s.controllers) {
app := appNames[n]
kind, class := "ingress/", "nginx"
if ocp {
kind, class = "route/", "openshift-router"
}
host := fmt.Sprintf("%s.apps.%s.example.com", app, name)
put(report.Workload{Kind: report.KindIngress, ID: newID(), Name: kind + app, Namespace: fmt.Sprintf("ns-%02d", n), State: report.StateOK,
Word: fmt.Sprintf("%s → app-%03d", host, n), Since: now.Add(-30 * 24 * time.Hour),
Facts: map[string]any{"hosts": host, "backends": fmt.Sprintf("app-%03d:8080", n), "tls": true, "class": class}})
}
clusterCerts(name, ocp, min(30, s.controllers), put)
if s.controllers > 0 {
put(svcRecord("gateway", "ns-00", clusterIP(ocp, k, 3999), lbAddr(k), "443", []string{"deployment/app-000"}, 3, now))
}
if ocp {
part := func(id, n, state, word string, facts map[string]any) {
put(report.Workload{Kind: report.KindCluster, ID: id, Name: n, State: state, Word: word, Since: now.Add(-20 * time.Minute), Facts: facts})
}
part("os-cv", "clusterversion/version", report.StateUpdating, "Updating to 4.20.4: Working towards 4.20.4: 512 of 873 done (58% complete)",
map[string]any{"version": "4.20.3", "desired": "4.20.4", "distribution": "OpenShift"})
part("os-mcp-master", "machineconfigpool/master", report.StateOK, "Up to date (3 nodes)", nil)
part("os-mcp-worker", "machineconfigpool/worker", report.StateUpdating, fmt.Sprintf("Updating: 12 of %d nodes done", s.nodes), nil)
for i, n := range []string{"authentication", "console", "dns", "etcd", "image-registry", "kube-apiserver", "machine-config", "network", "storage"} {
part(fmt.Sprintf("os-co-%d", i), "clusteroperator/"+n, report.StateOK, "Available", nil)
}
part("os-co-ingress", "clusteroperator/ingress", report.StateDegraded, `Degraded: IngressDegraded: The "default" ingress controller reports Degraded=True`, nil)
part("os-co-monitoring", "clusteroperator/monitoring", report.StateUpdating, "Progressing: Rolling out the stack.", nil)
}
if name == "cluster-00" { // the pod the probe changes
put(report.Workload{Kind: report.KindPod, ID: "probe", Name: "probe", Namespace: "fleet", Node: "node-00", State: report.StateOK, Word: "Running"})
}
return c
}
// The fake clusters' apps, one per namespace, and the VM groups.
var appNames = []string{"storefront", "checkout", "catalog", "payments", "search", "accounts", "orders", "inventory", "shipping", "pricing",
"reviews", "recommendations", "notifications", "billing", "ledger", "auth", "profile", "media", "reporting", "analytics",
"ingest", "scheduler", "returns", "loyalty", "fraud", "tax", "quotes", "support", "warehouse", "partners"}
var vmGroups = []struct{ name, port string }{{"erp", "8443"}, {"warehouse-db", "5432"}, {"file-servers", "445"}, {"directory", "636"},
{"build-agents", "22"}, {"legacy-crm", "8080"}, {"print", "631"}, {"backup-proxy", "8400"}, {"jump-hosts", "22"},
{"message-queue", "5671"}, {"license-servers", "27000"}, {"reporting-db", "1521"}}
func clusterIndex(name string) int {
n, _ := strconv.Atoi(strings.TrimPrefix(name, "cluster-"))
return n
}
// clusterIP is Service i's address: OpenShift's service network is
// 172.30.0.0/16, kubeadm's 10.96.0.0/12.
func clusterIP(openshift bool, k, i int) string {
if openshift {
return fmt.Sprintf("172.30.%d.%d", i/250, 10+i%250)
}
return fmt.Sprintf("10.%d.%d.%d", 96+k%16, i/250, 10+i%250)
}
// lbAddr is cluster k's load balancer, which some fleet hosts call on :443.
func lbAddr(k int) string { return fmt.Sprintf("10.200.%d.10", k%250) }
func svcRecord(name, ns, ip, external, ports string, backends []string, ready int, now time.Time) report.Workload {
w := report.Workload{Kind: report.KindService, ID: newID(), Name: name, Namespace: ns, State: report.StateOK,
Since: now.Add(-30 * 24 * time.Hour), Facts: map[string]any{"type": "ClusterIP", "cluster_ips": ip, "ports": ports}}
if external != "" {
w.Facts["type"], w.Facts["external"] = "LoadBalancer", external
}
if len(backends) > 0 {
w.Facts["backends"] = strings.Join(backends, ",")
}
switch ready {
case 0:
w.State, w.Word = report.StateStopped, "No ready endpoints"
case 1:
w.Word = "1 ready endpoint"
default:
w.Word = fmt.Sprintf("%d ready endpoints", ready)
}
return w
}
func (c *fakeCluster) addPod(ns, owner string, now time.Time) string {
id := newID()
c.recs[id] = report.Workload{Kind: report.KindPod, ID: id, Name: fmt.Sprintf("%s-%05x", strings.TrimPrefix(owner, "deployment/"), mrand.N(1<<20)),
Namespace: ns, Owner: owner, Node: fmt.Sprintf("node-%02d", mrand.N(max(c.shape.nodes, 1))), State: report.StateOK, Word: "Running",
Since: now.Add(-time.Duration(mrand.N(72)) * time.Hour), Labels: map[string]string{"app.kubernetes.io/name": strings.TrimPrefix(owner, "deployment/")},
Facts: map[string]any{"restarts": 0, "ready": "1/1"}}
c.pods = append(c.pods, id)
return id
}
// churn changes n pods: most restart once, the odd one is replaced.
func (c *fakeCluster) churn(n int, now time.Time) {
c.mu.Lock()
defer c.mu.Unlock()
for range n {
if len(c.pods) == 0 {
return
}
i := mrand.N(len(c.pods))
id := c.pods[i]
p := c.recs[id]
if p.State == report.StateCrashloop {
continue
}
if mrand.N(10) == 0 { // replaced: a new pod, the old one gone
delete(c.recs, id)
c.removed = append(c.removed, id)
c.pods = slices.Delete(c.pods, i, i+1)
nid := c.addPod(p.Namespace, p.Owner, now)
q := c.recs[nid]
q.State, q.Word, q.Since = report.StateStarting, "ContainerCreating", now
c.recs[nid] = q
c.changed[nid] = true
continue
}
r, _ := p.Facts["restarts"].(int)
p.Facts = map[string]any{"restarts": r + 1, "ready": "1/1", "last_exit": "OOMKilled"}
p.State, p.Word, p.Since = report.StateOK, "Running", now
c.recs[id] = p
c.changed[id] = true
}
// Pods started last round are running now.
for id, p := range c.recs {
if p.Kind == report.KindPod && p.State == report.StateStarting && now.Sub(p.Since) > 4*time.Second {
p.State, p.Word = report.StateOK, "Running"
c.recs[id] = p
c.changed[id] = true
}
}
}
func (c *fakeCluster) setNode(name, state, word string, now time.Time) {
c.mu.Lock()
defer c.mu.Unlock()
for id, r := range c.recs {
if r.Kind == report.KindNode && r.Name == name {
r.State, r.Word, r.Since = state, word, now
c.recs[id] = r
c.changed[id] = true
}
}
}
// probe sets the probe pod's word, and returns it.
func (c *fakeCluster) probe(now time.Time) string {
c.mu.Lock()
defer c.mu.Unlock()
c.seq++
p := c.recs["probe"]
p.Word, p.Since = fmt.Sprintf("Running (probe %d)", c.seq), now
c.recs["probe"] = p
c.changed["probe"] = true
return p.Word
}
// batches is what to send now: a full listing in pages, or the changes.
func (c *fakeCluster) batches(full bool) []report.Workloads {
out := c.listing(full)
for i := range out {
out[i].Source = c.source
}
return out
}
func (c *fakeCluster) listing(full bool) []report.Workloads {
c.mu.Lock()
defer c.mu.Unlock()
if !full {
b := report.Workloads{V: report.Version, Agent: "fleet", Generation: c.gen, Records: []report.Workload{}, Removed: c.removed}
for id := range c.changed {
if r, ok := c.recs[id]; ok {
b.Records = append(b.Records, r)
}
}
c.changed, c.removed = map[string]bool{}, nil
return []report.Workloads{b}
}
c.gen = newID()
all := make([]report.Workload, 0, len(c.recs))
for _, r := range c.recs {
all = append(all, r)
}
c.changed, c.removed = map[string]bool{}, nil
var out []report.Workloads
for len(all) > 0 {
n := min(len(all), 500)
out = append(out, report.Workloads{V: report.Version, Agent: "fleet", Generation: c.gen, Full: report.FullPage, Records: all[:n]})
all = all[n:]
}
if len(out) == 0 {
out = append(out, report.Workloads{V: report.Version, Agent: "fleet", Generation: c.gen})
}
out[0].Full = report.FullBegin
out = append(out, report.Workloads{V: report.Version, Agent: "fleet", Generation: c.gen, Full: report.FullEnd, Records: []report.Workload{}})
return out
}
func (c *fakeCluster) size() int {
c.mu.Lock()
defer c.mu.Unlock()
return len(c.recs)
}
// runCluster plays one cluster agent until ctx ends. A batch that fails is
// sent again before anything newer, as the agent does.
func runCluster(ctx context.Context, c *fakeCluster, client *http.Client, pick func() string, s *stats, probed func(word string, at time.Time)) {
start := time.Now()
var backlog []report.Workloads
send := func() {
for len(backlog) > 0 {
body, _ := json.Marshal(backlog[0])
t0 := time.Now()
req, _ := http.NewRequestWithContext(ctx, "POST", pick()+"/api/v1/agent/workloads", bytes.NewReader(body))
req.Header.Set("Authorization", "Bearer "+c.token)
req.Header.Set("Content-Type", "application/json")
resp, err := client.Do(req)
if ctx.Err() != nil {
return
}
if err != nil {
s.add(0, shortErr(err))
return
}
msg, _ := io.ReadAll(io.LimitReader(resp.Body, 300))
resp.Body.Close()
if resp.StatusCode/100 != 2 {
s.add(0, fmt.Sprintf("HTTP %d %s", resp.StatusCode, strings.TrimSpace(string(msg))))
return
}
s.add(time.Since(t0), "")
backlog = backlog[1:]
}
}
backlog = append(backlog, c.batches(true)...)
send()
lastList := time.Now()
var lastUsage time.Time
t := time.NewTicker(5 * time.Second)
defer t.Stop()
down := false
var probeWord string
var probeAt time.Time
for {
select {
case <-ctx.Done():
return
case now := <-t.C:
if c.shape.nodeDown > 0 && c.name == "cluster-00" {
el := now.Sub(start)
if !down && el >= c.shape.nodeDown && el < c.shape.nodeDown+c.shape.nodeDownFor {
down = true
c.setNode("node-01", report.StateDown, "Not ready (kubelet stopped posting)", now)
fmt.Printf("%s fleet: cluster-00 node-01 down\n", now.Format("15:04:05.000"))
} else if down && el >= c.shape.nodeDown+c.shape.nodeDownFor {
down = false
c.shape.nodeDown = 0
c.setNode("node-01", report.StateOK, "Ready", now)
fmt.Printf("%s fleet: cluster-00 node-01 back\n", now.Format("15:04:05.000"))
}
}
c.churn(int(c.shape.churn*5), now)
if probed != nil {
probeWord, probeAt = c.probe(now), now
}
if c.shape.relist > 0 && now.Sub(lastList) >= c.shape.relist {
backlog = append(backlog, c.batches(true)...)
lastList = now
} else {
backlog = append(backlog, c.batches(false)...)
}
// A usage reading a minute, as metrics.k8s.io gives the agent.
if (!c.noUsage || c.usageFn != nil) && now.Sub(lastUsage) >= time.Minute {
b := &backlog[len(backlog)-1]
b.Usage, b.UsageAt = c.usage(now), now.Truncate(time.Second)
if c.usageFn != nil {
b.Usage = c.usageFn(now)
}
lastUsage = now
}
send()
if probed != nil && len(backlog) == 0 {
probed(probeWord, probeAt)
}
}
}
}
// watchProbe times how long a probe change takes to show on each copy.
func watchProbe(ctx context.Context, client *http.Client, urls []string, cookies map[string]string, word string, at time.Time, s *stats) {
for _, u := range urls {
go func() {
for ctx.Err() == nil && time.Since(at) < 30*time.Second {
req, _ := http.NewRequestWithContext(ctx, "GET", u+"/api/v1/workloads?source=cluster-00&ns=fleet&q=probe", nil)
req.Header.Set("Cookie", cookies[u])
resp, err := client.Do(req)
if err == nil {
b, _ := io.ReadAll(resp.Body)
resp.Body.Close()
if bytes.Contains(b, []byte(`"`+word+`"`)) {
s.add(time.Since(at), "")
return
}
}
time.Sleep(50 * time.Millisecond)
}
if ctx.Err() == nil {
s.add(0, "not shown within 30 s on "+u)
}
}()
}
}
// signIn gets a dev sign-in cookie from each copy.
func signIn(ctx context.Context, client *http.Client, urls []string) (map[string]string, error) {
cookies := map[string]string{}
for _, u := range urls {
req, _ := http.NewRequestWithContext(ctx, "POST", u+"/auth/dev", nil)
req.Header.Set("X-Beacon", "1")
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("dev sign-in on %s: %w (is dev_signin on?)", u, err)
}
resp.Body.Close()
for _, c := range resp.Cookies() {
cookies[u] = c.Name + "=" + c.Value
}
if cookies[u] == "" {
return nil, fmt.Errorf("dev sign-in on %s: HTTP %d, no cookie (is dev_signin on?)", u, resp.StatusCode)
}
}
return cookies, nil
}
// guestOS is what KubeVirt's guest agent says: the name without the
// release's code name, "Red Hat Enterprise Linux 9.6".
func guestOS(os string) string {
name, _, _ := strings.Cut(os, " (")
return name
}
// usage is a reading for every node, controller and VM: each settles
// around its own level (from its id), drifting over the day, with a few
// running near their memory limit so the pages have something to show.
func (c *fakeCluster) usage(now time.Time) []report.Usage {
c.mu.Lock()
defer c.mu.Unlock()
day := math.Sin(2 * math.Pi * float64(now.Hour()*60+now.Minute()) / 1440) // -1 … 1 over the day
var out []report.Usage
for _, r := range c.recs {
h := fnv.New32a()
h.Write([]byte(r.ID))
seed := float64(h.Sum32()%1000) / 1000 // 0 … 1, fixed per record
level := 0.2 + 0.5*seed + 0.1*day + 0.05*(mrand.Float64()-0.5)
level = min(max(level, 0.02), 0.98)
// Memory moves less over the day than CPU, and sits elsewhere.
mem := min(max(0.3+0.45*float64(h.Sum32()/1000%1000)/1000+0.03*day, 0.05), 0.9)
switch r.Kind {
case report.KindNode:
if r.State != report.StateOK {
continue
}
cpus, gib := 32.0, 256.0 // a Kubernetes node's; Proxmox and ESXi nodes say their own
if n, ok := r.Facts["cpus"].(int); ok {
cpus = float64(n)
}
if g, ok := r.Facts["memory_gib"].(float64); ok {
gib = g
}
out = append(out, report.Usage{ID: r.ID, CPU: math.Round(cpus*level*1000) / 1000, Mem: int64(gib * mem * (1 << 30)), CPUOf: cpus, MemOf: int64(gib * (1 << 30))})
case report.KindController:
pods, _ := r.Facts["desired"].(int)
pods = max(pods, 1)
u := report.Usage{ID: r.ID, CPU: math.Round(float64(pods)*0.5*level*1000) / 1000, Mem: int64(float64(pods) * mem * (1 << 30)), Pods: pods}
if h.Sum32()%2 == 0 { // half have limits: half a core and 1 GiB a pod
u.CPUOf, u.MemOf = float64(pods)*0.5, int64(pods)<<30
if h.Sum32()%37 == 0 { // close to being OOM-killed
u.Mem = int64(float64(u.MemOf) * (0.92 + 0.05*mrand.Float64()))
}
}
out = append(out, u)
case report.KindVM:
if r.State == report.StateStopped {
continue
}
cpus, gib := 4.0, 8.0 // a KubeVirt VM's; a Proxmox guest says its own
if n, ok := r.Facts["vcpus"].(int); ok {
cpus = float64(n)
}
if g, ok := r.Facts["memory_gib"].(float64); ok {
gib = g
}
out = append(out, report.Usage{ID: r.ID, CPU: math.Round(cpus*level*1000) / 1000, Mem: int64(gib * mem * (1 << 30)), CPUOf: cpus, MemOf: int64(gib * (1 << 30)), Pods: 1})
}
}
return out
}