Files
cairnobs/scripts/fleet/demo.go
T
jcoffey-dev 1066ca9825
ci / check (pull_request) Successful in 6m58s
ci / postgres (pull_request) Successful in 10m29s
Demo fleet: four Windows servers, and names keep their capitals
fleet config -windows N adds up to four Windows servers (win-dc01,
win-fs01, win-sql01, win-hv01) with their services, event log, a
machine certificate in LocalMachine\My and, on win-hv01, twelve
Hyper-V VMs: a failed service, a critical event in the last hour, a
certificate ending soon, a saved VM and a failed one.

lowerFirst leaves a first word with another capital in it alone, so a
VM's alert says "Hyper-V reports a failure", not "hyper-V".

A Windows host without containers no longer shows an empty Containers
panel.
2026-11-15 03:36:47 -07:00

376 lines
14 KiB
Go

package main
import (
"bytes"
"fmt"
"log"
"os"
"path/filepath"
"time"
"github.com/cairnobs/cairnobs/internal/report"
)
// The demo's own integrations (`fleet config -demo`, `fleet run -demo`):
// the checks every site runs, the units and containers each host must
// keep running, and the backups each site keeps, so that every page has
// something to show. config writes the files obs-beacond reads; run makes
// the hosts report against them. Both work them out from the number of
// agents, so give both the same -n. Problems on purpose:
//
// checks legacy.example.com answers 502 (a warning-only check);
// billing.example.com's certificate ends in 9 days; the
// ping to the last site loses 3%
// units fleet-0041's redis-server has failed
// containers fleet-0007's app-04 exited; fleet-0008's app-09 is stopped
// on purpose
// backups site 1's newest pg-dump is 40 hours old; site 2's shrank to
// a third; site 3's backup disk is 93% full; site 4's
// pg-backup job failed; site 5's md1 is missing a disk
// role is what a host does in its site, by position (netmap.go).
func role(pos int) string {
switch {
case pos == 0:
return "gateway"
case pos <= 2:
return "lb"
case pos <= 29:
return "app"
case pos <= 39:
return "db"
case pos <= 44:
return "cache"
}
return "mq"
}
// demoUnits are the systemd units host n must keep active.
func demoUnits(n int) []string {
base := []string{"ssh", "chrony"}
switch role(n % siteSize) {
case "gateway":
return append(base, "wg-quick@wg0")
case "lb":
return append(base, "haproxy")
case "app":
return append(base, "docker")
case "db":
return append(base, "postgresql")
case "cache":
return append(base, "redis-server")
}
return append(base, "nats-server")
}
const (
demoFailedUnit = 41 // its redis-server has failed
demoExited = 7 // its app-04 exited
demoStoppedOnPurp = 8 // its app-09 is stopped on purpose
)
// demoCheck is one check, where it runs from, and how it goes.
type demoCheck struct {
id, name, yaml string // yaml: the check's own lines, after id and name
from []int
result func(from int, now time.Time) report.CheckResult
}
func gateways(hosts int) []int {
var g []int
for s := range (hosts + siteSize - 1) / siteSize {
g = append(g, s*siteSize)
}
return g
}
func demoChecks(hosts int) []demoCheck {
gws := gateways(hosts)
web := gws[:min(3, len(gws))]
ok := func(id string, ms float64, f func(*report.CheckResult)) func(int, time.Time) report.CheckResult {
return func(from int, now time.Time) report.CheckResult {
r := report.CheckResult{ID: id, At: now.UTC(), OK: true, Ms: ms * (0.8 + 0.4*float64(from%7)/6)}
if f != nil {
f(&r)
}
return r
}
}
cert := func(days int) func(*report.CheckResult) {
return func(r *report.CheckResult) {
r.Status = 200
r.CertExpires = time.Now().Add(time.Duration(days) * 24 * time.Hour).Truncate(time.Hour).UTC()
}
}
var out []demoCheck
for _, w := range []struct {
id, host string
days int
}{{"web-shop", "shop.example.com", 61}, {"web-api", "api.example.com", 61}, {"web-docs", "docs.example.com", 74}, {"web-status", "status.example.com", 45}, {"web-billing", "billing.example.com", 9}} {
out = append(out, demoCheck{id: w.id, name: w.host, yaml: fmt.Sprintf("http: https://%s/", w.host), from: web, result: ok(w.id, 120, cert(w.days))})
}
out = append(out,
demoCheck{id: "web-legacy", name: "legacy.example.com", yaml: "http: https://legacy.example.com/\n severity: warning", from: web,
result: func(from int, now time.Time) report.CheckResult {
return report.CheckResult{ID: "web-legacy", At: now.UTC(), Status: 502, Ms: 2100, Error: "HTTP 502, want 200"}
}},
demoCheck{id: "dns-mx", name: "example.com MX", yaml: "dns: example.com\n type: MX\n contains: mx1.example.com", from: web,
result: ok("dns-mx", 12, func(r *report.CheckResult) { r.Answer = "10 mx1.example.com." })},
demoCheck{id: "smtp", name: "mx1.example.com SMTP", yaml: "tcp: mx1.example.com:25\n contains: ESMTP", from: web,
result: ok("smtp", 35, func(r *report.CheckResult) { r.Answer = "220 mx1.example.com ESMTP ready" })},
demoCheck{id: "submission", name: "mail.example.com submission", yaml: "tcp: mail.example.com:587\n contains: ESMTP", from: web,
result: ok("submission", 38, func(r *report.CheckResult) { r.Answer = "220 mail.example.com ESMTP ready" })},
demoCheck{id: "imaps", name: "mail.example.com IMAPS", yaml: "tcp: mail.example.com:993", from: web, result: ok("imaps", 41, nil)},
demoCheck{id: "rbl-mx1", name: "mx1.example.com on block lists", yaml: "rbl: 192.0.2.25", from: web[:1],
result: ok("rbl-mx1", 180, func(r *report.CheckResult) { r.Answer = "clean on 5 of 5" })},
// mx2 landed on a list that flags whole networks: a warning to look at, not a page.
demoCheck{id: "rbl-mx2", name: "mx2.example.com on block lists", yaml: "rbl: 192.0.2.26\n severity: warning", from: web[:1],
result: func(from int, now time.Time) report.CheckResult {
return report.CheckResult{ID: "rbl-mx2", At: now.UTC(), Ms: 210, Answer: "clean on 4 of 5",
Error: "listed on dnsbl-1.uceprotect.net (127.0.0.2)"}
}},
)
// Every site's gateway pings every other's over WireGuard.
for s, gw := range gws {
var from []int
for _, o := range gws {
if o != gw {
from = append(from, o)
}
}
if len(from) == 0 {
continue
}
id := fmt.Sprintf("ping-site-%02d", s)
last := s == len(gws)-1 && s > 0
out = append(out, demoCheck{id: id, name: fmt.Sprintf("site %d over WireGuard", s), yaml: "ping: " + siteAddr(s, 0) + "\n severity: warning", from: from,
result: func(from int, now time.Time) report.CheckResult {
d := from/siteSize - s
if d < 0 {
d = -d
}
r := report.CheckResult{ID: id, At: now.UTC(), OK: true, Ms: 0.4 + 9.5*float64(d)}
if last {
r.Loss = 3
}
return r
}})
}
return out
}
// demoBackups: in each of the first six sites the gateway keeps the
// backups of the site's first four databases.
type demoBackup struct {
store int
senders []int
}
func demoBackupSites(hosts int) []demoBackup {
var out []demoBackup
for s := range min(6, (hosts+siteSize-1)/siteSize) {
b := demoBackup{store: s * siteSize}
for p := 30; p < 34; p++ {
if s*siteSize+p < hosts {
b.senders = append(b.senders, s*siteSize+p)
}
}
if len(b.senders) > 0 {
out = append(out, b)
}
}
return out
}
// demoScan is host n's backup scan, or nil.
func demoScan(n, hosts int, now time.Time) *report.Backups {
for _, b := range demoBackupSites(hosts) {
site := b.store / siteSize
if n == b.store {
sc := &report.Backups{At: now.Add(-7 * time.Minute).UTC(), Jobs: []report.BackupJob{}, Made: []report.BackupDir{},
RAID: []report.RAIDArray{{Dev: "md0", State: "[UU]"}, {Dev: "md1", State: "[UU]"}},
SMART: []report.DiskHealth{{Disk: "sda", OK: true}, {Disk: "sdb", OK: true}}}
used := 41 + float64(site*7)
if site == 3 {
used = 93
}
if site == 5 {
sc.RAID[1].State = "[U_]"
}
sc.Roots = []report.BackupRoot{{Path: "/srv/backups", UsedPct: used}}
for k, from := range b.senders {
for _, name := range []string{"pg-dump", "system"} {
size := int64(3+k) << 30
if name == "system" {
size = int64(800+40*k) << 20
}
d := report.BackupDir{Name: name, From: hostName(from), Path: fmt.Sprintf("/srv/backups/%s/%s/%s", hostName(from), name, now.Format("2006-01-02")),
Size: size, PrevSize: size - size/50, Count: 14, Modified: now.Add(-time.Duration(3+k) * time.Hour).UTC()}
if k == 0 && name == "pg-dump" {
switch site {
case 1:
d.Modified = now.Add(-40 * time.Hour).UTC()
case 2:
d.PrevSize, d.Size = d.Size, d.Size/3
}
}
sc.Received = append(sc.Received, d)
}
}
return sc
}
for k, from := range b.senders {
if n != from {
continue
}
j := report.BackupJob{Unit: "pg-backup.service", Result: "success", Active: "inactive", Finished: now.Add(-time.Duration(3+k) * time.Hour).UTC()}
if site == 4 && k == 0 {
j.Result = "exit-code"
}
return &report.Backups{At: now.Add(-5 * time.Minute).UTC(), Received: []report.BackupDir{}, Made: []report.BackupDir{}, Roots: []report.BackupRoot{},
Jobs: []report.BackupJob{j}, RAID: []report.RAIDArray{}, SMART: []report.DiskHealth{{Disk: "nvme0n1", OK: true}}}
}
}
return nil
}
// writeDemo writes the demo's files into dir, and demo.yaml: the lines to
// add to obs-beacond.yaml.
func writeDemo(dir string, hosts, windows int) {
var checks bytes.Buffer
checks.WriteString("# The fleet demo's checks (fleet config -demo).\nchecks:\n")
for _, c := range demoChecks(hosts) {
fmt.Fprintf(&checks, " - id: %s\n name: %q\n %s\n from: [", c.id, c.name, c.yaml)
for k, f := range c.from {
if k > 0 {
checks.WriteString(", ")
}
checks.WriteString(hostName(f))
}
checks.WriteString("]\n")
}
var exp bytes.Buffer
exp.WriteString("# The fleet demo's hosts: what must keep running (fleet config -demo).\nhosts:\n")
for n := range hosts {
fmt.Fprintf(&exp, " %s:\n running: [\"app-*\"]\n units: [", hostName(n))
for k, u := range demoUnits(n) {
if k > 0 {
exp.WriteString(", ")
}
exp.WriteString(u)
}
exp.WriteString("]\n")
if n == demoStoppedOnPurp {
exp.WriteString(" stopped:\n - match: app-09\n note: Replaced by app-09b; kept for a rollback until Friday.\n")
}
}
exp.WriteString(winExpected(windows))
if hosts > ibmiHost {
// The IBM i partitions' subsystems that must be active (run -ibmi):
// prod2's ORDERS isn't, and ha1, the standby, expects none.
src := hostName(ibmiHost)
fmt.Fprintf(&exp, "clusters:\n %s:prod1:\n subsystems: [QINTER, QBATCH, QUSRWRK, ORDLIB/ORDERS]\n", src)
fmt.Fprintf(&exp, " %s:prod2:\n subsystems: [QINTER, QBATCH, QUSRWRK, ORDLIB/ORDERS]\n", src)
fmt.Fprintf(&exp, " %s:dev1:\n subsystems: [QINTER, QBATCH]\n", src)
}
var bk bytes.Buffer
bk.WriteString("# The fleet demo's backups (fleet config -demo).\nmax_age: 26h\nhosts:\n")
for _, b := range demoBackupSites(hosts) {
fmt.Fprintf(&bk, " %s:\n received:\n", hostName(b.store))
for _, s := range b.senders {
fmt.Fprintf(&bk, " - {from: %s, root: /srv/backups/%s, expect: [pg-dump, system]}\n", hostName(s), hostName(s))
}
for _, s := range b.senders {
fmt.Fprintf(&bk, " %s:\n jobs: [pg-backup.service]\n", hostName(s))
}
}
abs, err := filepath.Abs(dir)
if err != nil {
log.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "topology.yaml"), []byte(demoTopology), 0o644); err != nil {
log.Fatal(err)
}
conf := fmt.Sprintf("# The fleet demo (fleet config -demo): add these to obs-beacond.yaml.\nchecks: %s\nexpected: %s\nbackups: %s\ntopology: %s\n",
filepath.Join(abs, "checks.yaml"), filepath.Join(abs, "expected.yaml"), filepath.Join(abs, "backups.yaml"), filepath.Join(abs, "topology.yaml"))
if hasDataLayer(hosts) {
conf += dataLayerYAML()
}
for name, b := range map[string][]byte{"checks.yaml": checks.Bytes(), "expected.yaml": exp.Bytes(), "backups.yaml": bk.Bytes(), "demo.yaml": []byte(conf)} {
if err := os.WriteFile(filepath.Join(dir, name), b, 0o644); err != nil {
log.Fatal(err)
}
}
}
// demoHost is set when the demo runs: what host n reports beside the rest.
type demoHost struct {
n, hosts int
checks []demoCheck
}
func newDemoHost(n, hosts int, all []demoCheck) *demoHost {
d := &demoHost{n: n, hosts: hosts}
for _, c := range all {
for _, f := range c.from {
if f == n {
d.checks = append(d.checks, c)
}
}
}
return d
}
// setup changes the host's containers for the demo's problems.
func (d *demoHost) setup(r *report.Report) {
if r.Containers == nil {
return
}
for k := range r.Containers.List {
c := &r.Containers.List[k]
switch {
case d.n == demoExited && c.Name == "app-04":
c.State, c.Status, c.Health = "exited", "Exited (137) 12 minutes ago", ""
case d.n == demoStoppedOnPurp && c.Name == "app-09":
c.State, c.Status, c.Health = "exited", "Exited (0) 2 days ago", ""
}
}
}
// stamp adds the units, check results and backup scan to a report.
func (d *demoHost) stamp(r *report.Report, start, now time.Time) {
r.Units = nil
for _, u := range demoUnits(d.n) {
un := report.Unit{Name: u + ".service", Load: "loaded", Active: "active", Sub: "running", Since: start.UTC()}
if d.n == demoFailedUnit && u == "redis-server" {
un.Active, un.Sub, un.Restarts, un.Since = "failed", "failed", 5, time.Time{}
}
r.Units = append(r.Units, un)
}
r.Checks = nil
for _, c := range d.checks {
r.Checks = append(r.Checks, c.result(d.n, now))
}
r.Backups = demoScan(d.n, d.hosts, now)
if hasDataLayer(d.hosts) {
r.Fetches = dataFetches(d.n, now)
}
}
// demoTopology names the demo's services: what each claims, what it needs
// (lines on the service map; alerts group by them) and the checks that
// judge it. The Mail page shows the mail service's checks.
const demoTopology = `# The fleet demo's services (fleet config -demo). Format: docs/formats.md, "topology.yaml".
services:
- {id: cloudflare, name: Cloudflare, tier: edge, cloudflare: true}
- {id: shop, name: Web shop, tier: public, needs: [cloudflare, apps], checks: [web-shop, web-billing]}
- {id: api, name: Public API, tier: public, needs: [cloudflare, apps], checks: [web-api]}
- {id: sites, name: Docs and status, tier: public, needs: [cloudflare], checks: [web-docs, web-status, web-legacy]}
- {id: mail, name: Mail, tier: public, needs: [postgres], checks: [dns-mx, smtp, submission, imaps, rbl-mx1, rbl-mx2]}
- {id: apps, name: App servers, tier: apps, needs: [haproxy, redis, nats], units: ["*/docker"], containers: ["*/app-*"]}
- {id: haproxy, name: HAProxy, tier: data, needs: [postgres], units: ["*/haproxy"]}
- {id: postgres, name: PostgreSQL (Patroni), tier: data, units: ["*/postgresql"]}
- {id: redis, name: Redis, tier: data, units: ["*/redis-server"]}
- {id: nats, name: NATS, tier: data, units: ["*/nats-server"]}
`