release / release (push) Successful in 7m6s
The new Cairn OBS: a self-hosted monitor for networks, hosts and services. Licensed AGPLv3.
180 lines
9.5 KiB
Go
180 lines
9.5 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"math"
|
|
"math/rand/v2"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/cairnobs/cairnobs/internal/report"
|
|
)
|
|
|
|
// FlashArrays, as a host agent with a pure: section sends them
|
|
// (docs/pure.md): fleet-0040's agent reads them all, each a source of its
|
|
// own.
|
|
const pureHost = 40 // the fleet host whose agent reads them
|
|
|
|
// pureArrays are the arrays, in the order -pure N takes them.
|
|
var pureArrays = []string{"fa-prod-01", "fa-prod-02", "fa-dr-01", "fa-vdi-01", "fb-prod-01"}
|
|
|
|
// newPure is the k'th array: a FlashArray, or the FlashBlade last.
|
|
func newPure(name, token string, k int) *fakeCluster {
|
|
if strings.HasPrefix(pureArrays[k], "fb-") {
|
|
return newFlashBlade(name, token, k)
|
|
}
|
|
return newFlashArray(name, token, k, 600)
|
|
}
|
|
|
|
// newFlashBlade is fb-prod-01 (docs/pure.md, step 5): ten blades, one
|
|
// evacuating; a failed fan; 400 file systems, cluster-02's first 100 VM
|
|
// disks among them (Portworx names them after their PersistentVolumes),
|
|
// one whose replication to fb-dr-01 is unhealthy; 40 buckets.
|
|
func newFlashBlade(name, token string, k int) *fakeCluster {
|
|
arr := pureArrays[k]
|
|
c := &fakeCluster{name: name, token: token, recs: map[string]report.Workload{}, changed: map[string]bool{}, noUsage: true, source: arr}
|
|
c.usageFn = func(now time.Time) []report.Usage { return pureIO(k, now) }
|
|
now := time.Now()
|
|
put := func(w report.Workload) { c.recs[w.ID] = w }
|
|
put(report.Workload{Kind: report.KindCluster, ID: report.ClusterSelf, Name: arr, State: report.StateDegraded, Word: "Hardware: CH1.FAN2 critical (Fan speed is 0 RPM)",
|
|
Since: now.Add(-95 * time.Minute), Facts: map[string]any{"distribution": "Purity//FB", "version": "4.5.6", "model": "FlashBlade//S", "capacity_tib": 2000.0,
|
|
"used_pct": 61.0, "data_reduction": 3.1, "hardware": "CH1.FAN2 critical (Fan speed is 0 RPM)", "alerts": "CH1.FAN2: Fan failure (critical)"}})
|
|
for i := 1; i <= 10; i++ {
|
|
r := report.Workload{Kind: report.KindNode, ID: fmt.Sprintf("ch1.fb%d", i), Name: fmt.Sprintf("CH1.FB%d", i), State: report.StateOK, Word: "Healthy",
|
|
Since: now.Add(-200 * 24 * time.Hour), Facts: map[string]any{"raw_tib": 48.0}}
|
|
if i == 6 {
|
|
r.State, r.Word, r.Since = report.StateUpdating, "Evacuating its data (37%)", now.Add(-2*time.Hour)
|
|
}
|
|
put(r)
|
|
}
|
|
put(report.Workload{Kind: report.KindCluster, ID: "connection/c-fb-dr-01", Name: "connection/fb-dr-01", State: report.StateOK, Word: "Connected",
|
|
Since: now.Add(-90 * 24 * time.Hour), Facts: map[string]any{"type": "async-replication", "version": "4.5.6"}})
|
|
for n := range 400 {
|
|
fname := fmt.Sprintf("share-%03d", n)
|
|
if n < 100 {
|
|
fname = "px_8f3a2b1c-" + pvName("cluster-02", n)
|
|
}
|
|
r := report.Workload{Kind: report.KindPVC, ID: fmt.Sprintf("%08x-0000-4000-8000-%012d", k, n), Name: fname, State: report.StateOK,
|
|
Word: "Shared over NFSv4.1", Since: now.Add(-time.Duration(1+n%300) * 24 * time.Hour),
|
|
Facts: map[string]any{"type": "file system", "provisioned_gib": float64(int(100) << (n % 5)), "used_gib": float64(20 + n%70), "protocols": "NFSv4.1"}}
|
|
switch {
|
|
case n == 100:
|
|
r.Name, r.Word, r.Facts["protocols"] = "home", "Shared over NFSv3, NFSv4.1, SMB", "NFSv3, NFSv4.1, SMB"
|
|
r.Facts["replication"] = "to fb-dr-01, replicating, 4 min behind"
|
|
case n == 101:
|
|
r.Name, r.State = "scratch", report.StateDegraded
|
|
r.Word = "Replication to fb-dr-01 is unhealthy: Target file system is full"
|
|
r.Facts["replication"] = "to fb-dr-01, unhealthy, 5 h behind"
|
|
case n%50 == 7:
|
|
r.Word = "Not shared over any protocol"
|
|
delete(r.Facts, "protocols")
|
|
}
|
|
put(r)
|
|
}
|
|
for n := range 40 {
|
|
r := report.Workload{Kind: report.KindPVC, ID: fmt.Sprintf("%08x-0001-4000-8000-%012d", k, n), Name: fmt.Sprintf("bucket-%02d", n), State: report.StateOK,
|
|
Word: "Ready", Since: now.Add(-time.Duration(1+n*5) * 24 * time.Hour),
|
|
Facts: map[string]any{"type": "bucket", "account": []string{"veeam", "ops", "web", "ml"}[n%4], "used_gib": float64(500 + n*37), "objects": 1000 * (n + 1)}}
|
|
if n == 0 {
|
|
r.Name, r.Facts["replication"], r.Facts["versioning"] = "backups", "to s3-offsite, paused", "enabled"
|
|
}
|
|
put(r)
|
|
}
|
|
return c
|
|
}
|
|
|
|
func newFlashArray(name, token string, k, volumes int) *fakeCluster {
|
|
arr := pureArrays[k]
|
|
c := &fakeCluster{name: name, token: token, recs: map[string]report.Workload{}, changed: map[string]bool{}, noUsage: true, source: arr}
|
|
c.usageFn = func(now time.Time) []report.Usage { return pureIO(k, now) }
|
|
now := time.Now()
|
|
put := func(w report.Workload) { c.recs[w.ID] = w }
|
|
self := report.Workload{Kind: report.KindCluster, ID: report.ClusterSelf, Name: arr, State: report.StateOK, Word: "API answering",
|
|
Since: now.Add(-120 * 24 * time.Hour), Facts: map[string]any{"distribution": "Purity//FA", "version": "6.7.3", "capacity_tib": 500.0,
|
|
"used_pct": float64([]int{71, 68, 54, 83}[k]), "data_reduction": []float64{4.2, 4.1, 5.3, 9.8}[k], "model": "FA-X70R4"}}
|
|
if arr == "fa-prod-01" {
|
|
self.State, self.Word = report.StateDegraded, "Hardware: CH0.BAY7 failed (Drive reported a media error)"
|
|
self.Facts["hardware"], self.Facts["alerts"] = "CH0.BAY7 failed (Drive reported a media error)", "CH0.BAY7: Drive failure (critical)"
|
|
self.Since = now.Add(-3 * time.Hour)
|
|
}
|
|
put(self)
|
|
for n, mode := range []string{"primary", "secondary"} {
|
|
r := report.Workload{Kind: report.KindNode, ID: fmt.Sprintf("ct%d", n), Name: fmt.Sprintf("CT%d", n), State: report.StateOK, Word: "Ready, " + mode,
|
|
Since: now.Add(-40 * 24 * time.Hour), Facts: map[string]any{"mode": mode, "model": "FA-X70R4", "version": "6.7.3"}}
|
|
if arr == "fa-vdi-01" && n == 1 {
|
|
r.State, r.Word, r.Since = report.StateUpdating, "Updating", now.Add(-12*time.Minute)
|
|
}
|
|
put(r)
|
|
}
|
|
switch arr {
|
|
case "fa-prod-01", "fa-prod-02":
|
|
peer := map[string]string{"fa-prod-01": "fa-prod-02", "fa-prod-02": "fa-prod-01"}[arr]
|
|
put(report.Workload{Kind: report.KindCluster, ID: "pod/p-prod", Name: "pod/prod-pod", State: report.StateOK, Word: "In sync",
|
|
Since: now.Add(-60 * 24 * time.Hour), Facts: map[string]any{"arrays": 2}})
|
|
put(report.Workload{Kind: report.KindCluster, ID: "connection/c-" + peer, Name: "connection/" + peer, State: report.StateOK, Word: "Connected",
|
|
Since: now.Add(-60 * 24 * time.Hour), Facts: map[string]any{"type": "sync-replication", "version": "6.7.3"}})
|
|
put(report.Workload{Kind: report.KindCluster, ID: "connection/c-fa-dr-01", Name: "connection/fa-dr-01", State: report.StateOK, Word: "Connected",
|
|
Since: now.Add(-60 * 24 * time.Hour), Facts: map[string]any{"type": "async-replication", "version": "6.7.3"}})
|
|
case "fa-dr-01":
|
|
put(report.Workload{Kind: report.KindCluster, ID: "connection/c-fa-prod-01", Name: "connection/fa-prod-01", State: report.StateDegraded,
|
|
Word: "Connected, but its paths aren't redundant", Since: now.Add(-50 * time.Minute), Facts: map[string]any{"type": "async-replication", "version": "6.7.3"}})
|
|
}
|
|
prefix := map[string]string{"fa-prod-01": "k8s-pvc", "fa-prod-02": "k8s-pvc", "fa-dr-01": "dr-copy", "fa-vdi-01": "vdi"}[arr]
|
|
for n := range volumes {
|
|
vname := fmt.Sprintf("%s-%04d", prefix, n)
|
|
// cluster-00's VM disks are on fa-prod-01, cluster-01's in the pod
|
|
// both prod arrays share: named after their PersistentVolumes.
|
|
if cl := map[string]string{"fa-prod-01": "cluster-00", "fa-prod-02": "cluster-01"}[arr]; cl != "" && n < 300 {
|
|
vname = "k8s-" + pvName(cl, n)
|
|
if cl == "cluster-01" {
|
|
vname = "prod-pod::" + vname
|
|
}
|
|
}
|
|
r := report.Workload{Kind: report.KindPVC, ID: fmt.Sprintf("%08x-0000-4000-8000-%012d", k, n), Name: vname,
|
|
State: report.StateOK, Word: "Connected", Since: now.Add(-time.Duration(1+n%200) * 24 * time.Hour),
|
|
Facts: map[string]any{"type": "volume", "provisioned_gib": float64(int(50) << (n % 4)), "used_gib": float64(10 + n%40), "hosts": 1 + n%3}}
|
|
switch {
|
|
case arr == "fa-dr-01":
|
|
r.Word = "Demoted: a replication target, read only"
|
|
case n%97 == 13:
|
|
r.Word, r.Facts["hosts"] = "Not connected to a host", 0
|
|
case arr == "fa-prod-01" && n == 42:
|
|
r.State, r.Word, r.Facts["alerts"] = report.StateDegraded, "Alert from Purity: Volume space exceeded", "Volume space exceeded (critical)"
|
|
}
|
|
if arr != "fa-vdi-01" && arr != "fa-dr-01" && n%5 == 0 {
|
|
r.Facts["pod"] = "prod-pod"
|
|
}
|
|
put(r)
|
|
}
|
|
return c
|
|
}
|
|
|
|
// pureIO is an array's performance a minute, as /arrays/performance gives
|
|
// it: busy in the working day, quiet at night, with noise; fa-prod-01's
|
|
// writes slow while its failed drive's data is rebuilt, and fa-dr-01 is
|
|
// written in bursts as replication lands.
|
|
func pureIO(k int, now time.Time) []report.Usage {
|
|
base := []float64{62000, 48000, 9000, 31000, 140000}[k]
|
|
day := 0.55 + 0.45*math.Sin(2*math.Pi*(float64(now.Hour()*60+now.Minute())/1440-0.3)) // peaks mid-afternoon
|
|
noise := func() float64 { return 0.9 + 0.2*rand.Float64() }
|
|
ops := base * day * noise()
|
|
readShare := []float64{0.68, 0.7, 0.15, 0.8, 0.75}[k]
|
|
readUs, writeUs := 250+60*day*noise(), 120+40*day*noise()
|
|
if k == 0 {
|
|
writeUs *= 2.4 // rebuilding
|
|
}
|
|
if k == 2 && now.Minute()%15 < 3 {
|
|
ops *= 4 // a replication burst
|
|
}
|
|
capacity := int64(500) << 40
|
|
if k == 4 {
|
|
capacity = 2000 << 40 // the FlashBlade
|
|
readUs, writeUs = readUs*3, writeUs*8
|
|
}
|
|
used := float64([]int{71, 68, 54, 83, 61}[k])/100 + float64(now.Unix()%86400)/86400*0.004 // grows a little through the day
|
|
return []report.Usage{{ID: report.ClusterSelf, IO: &report.IO{
|
|
ReadOps: math.Round(ops * readShare), WriteOps: math.Round(ops * (1 - readShare)), ReadUs: math.Round(readUs), WriteUs: math.Round(writeUs),
|
|
ReadBps: math.Round(ops * readShare * 32 * 1024), WriteBps: math.Round(ops * (1 - readShare) * 24 * 1024),
|
|
Used: int64(used * float64(capacity)), UsedOf: capacity}}}
|
|
}
|