Files
cairnobs/scripts/fleet/datalayer.go
T
jcoffey-dev 95965ed5e8
ci / postgres (pull_request) Successful in 3m14s
ci / check (pull_request) Successful in 8m22s
Demo: hosts named for their site and role, collectors on their own
Things that gave the simulator away (#40):

- hosts were fleet-0000 to fleet-0119. Now they're named for their site
  and what they do: phx1-gw01, phx1-lb01, phx1-app05, phx1-db03,
  phx1-cache02, iad1-lb02, lon1-kvm01 (collectors.go);
- fleet-0041 was a Redis cache, a KVM host, the IBM i collector and the
  HMC reader at once, so IBM i alerts read as if PROD2 ran on it. The
  agents that read storage, IBM i and virtualization are small hosts of
  their own now: phx1-stor01 (the Pure arrays), phx1-ibmi01 (the
  partitions and the HMC), phx1-virt01 (vCenter, Proxmox, Nutanix).
  fleet config writes their agents;
- libvirt runs only on app servers, not on a load balancer or a cache;
- RHEL hosts' units are sshd and chronyd, as RHEL names them;
- Akamai's CP codes no longer have round counts.

Varied disks and jittered readings came with #39 (load.go).
2026-11-15 23:20:21 -07:00

181 lines
6.3 KiB
Go

package main
import (
"context"
"fmt"
"log"
"net"
"net/http"
"strings"
"time"
"github.com/cairnobs/cairnobs/internal/report"
)
// The demo's data layer (Data layer page, docs/formats.md): site 0's first
// three databases are a Patroni cluster, its load balancers run HAProxy in
// front of it, its NATS hosts a three-node cluster, and its caches a Garage
// cluster. Each Patroni member answers on its own loopback address
// (pgAddr), so obs-beacond can match a member to its API as it does over a
// real network; the status pages come in the hosts' reports, as an agent's
// fetches do.
// Problems on purpose: the leader is phx1-db02, not its usual phx1-db01
// (a failover nobody moved back), and every so often phx1-db03 falls 40
// MB behind and HAProxy takes it out of the replicas' pool until it
// catches up (scenario.go, story 2).
const patroniPort = 58420
var (
pgMembers = []int{30, 31, 32}
pgLeader = 31
pgHome = 30
pgLagging = 32
natsHosts = []int{45, 46, 47}
garageHosts = []int{40, 41, 42}
etcdHosts = pgMembers
haproxyLBs = []int{1, 2}
)
// hasDataLayer: the fleet has every host the data layer names.
func hasDataLayer(hosts int) bool { return hosts > 47 }
// dataLayerYAML is the datalayer: section for obs-beacond.yaml.
func dataLayerYAML() string {
names := func(ns []int) string {
var s []string
for _, n := range ns {
s = append(s, hostName(n))
}
return strings.Join(s, ", ")
}
var urls []string
for _, m := range pgMembers {
urls = append(urls, fmt.Sprintf("http://%s:%d", pgAddr(m), patroniPort))
}
return fmt.Sprintf(`datalayer:
patroni: [%s]
sync_standby: true
preferred_leader: %s
probes:
- {id: haproxy, kind: haproxy, url: "http://127.0.0.1:8404/stats;csv", from: [%s], exactly_one: [pg-primary], at_least_one: [pg-replicas]}
- {id: nats, kind: nats, url: "http://127.0.0.1:8222/varz", from: [%s], peers: 2}
- {id: garage, kind: garage, url: "http://127.0.0.1:3903/health", from: [%s]}
- {id: etcd, kind: etcd, url: "http://127.0.0.1:2381/metrics", from: [%s], fsync_warn_ms: 500}
`, strings.Join(urls, ", "), hostName(pgHome), names(haproxyLBs), names(natsHosts), names(garageHosts), names(etcdHosts))
}
// pgAddr is member n's loopback address.
func pgAddr(n int) string { return fmt.Sprintf("127.0.84.%d", n) }
// servePatroni answers /cluster and /patroni for each member.
func servePatroni(ctx context.Context) {
for _, me := range pgMembers {
ln, err := net.Listen("tcp", fmt.Sprintf("%s:%d", pgAddr(me), patroniPort))
if err != nil {
log.Fatal(err)
}
srv := &http.Server{ReadHeaderTimeout: 5 * time.Second, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch r.URL.Path {
case "/cluster":
var ms []string
for _, m := range pgMembers {
role, state, lag := "replica", "streaming", `,"lag":0`
switch m {
case pgLeader:
role, state, lag = "leader", "running", ""
case pgHome:
role = "sync_standby"
case pgLagging:
if spellReplicaBehind.active(time.Now()) {
lag = `,"lag":41943040`
}
}
ms = append(ms, fmt.Sprintf(`{"name":%q,"role":%q,"state":%q,"host":%q,"port":5432,"timeline":7%s}`, hostName(m), role, state, pgAddr(m), lag))
}
fmt.Fprintf(w, `{"members":[%s],"scope":"shop-pg"}`, strings.Join(ms, ","))
case "/patroni":
role := "replica"
if me == pgLeader {
role = "primary"
}
fmt.Fprintf(w, `{"state":"running","role":%q,"dcs_last_seen":%d,"timeline":7}`, role, time.Now().Unix()-3)
default:
http.NotFound(w, r)
}
})}
go func() { _ = srv.Serve(ln) }()
go func() { <-ctx.Done(); _ = srv.Close() }()
}
log.Printf("fleet: Patroni cluster shop-pg on %s-%d port %d", pgAddr(pgMembers[0]), pgMembers[len(pgMembers)-1], patroniPort)
}
// dataFetches are the status pages host n reads, if any (site 0 only).
func dataFetches(n int, now time.Time) []report.Fetch {
if n >= siteSize {
return nil
}
at := now.Add(-2 * time.Second).UTC()
var out []report.Fetch
for _, lb := range haproxyLBs {
if n != lb {
continue
}
var b strings.Builder
b.WriteString("# pxname,svname,qcur,status,weight\nstats,FRONTEND,,OPEN,\n")
backend := func(name string, up func(m int) bool, servers []int) {
fmt.Fprintf(&b, "%s,FRONTEND,,OPEN,\n", name)
for _, m := range servers {
st := "DOWN"
if up(m) {
st = "UP"
}
fmt.Fprintf(&b, "%s,%s,0,%s,1\n", name, hostName(m), st)
}
fmt.Fprintf(&b, "%s,BACKEND,0,UP,1\n", name)
}
backend("pg-primary", func(m int) bool { return m == pgLeader }, pgMembers)
behind := spellReplicaBehind.active(now)
backend("pg-replicas", func(m int) bool { return m != pgLeader && !(behind && m == pgLagging) }, pgMembers)
var apps []int
for p := 3; p < 13; p++ {
apps = append(apps, p)
}
backend("shop-web", func(int) bool { return true }, apps)
out = append(out, report.Fetch{ID: "haproxy", At: at, Status: 200, Body: b.String()})
}
for k, m := range natsHosts {
if n == m {
out = append(out, report.Fetch{ID: "nats", At: at, Status: 200, Body: fmt.Sprintf(
`{"server_name":%q,"version":"2.11.17","connections":%d,"routes":2,"remotes":2,"slow_consumers":0,"in_msgs":%d,"out_msgs":%d,"uptime":"21d4h"}`,
hostName(m), 40+k*3, 1_900_000+k*1000, 2_400_000+k*1000)})
}
}
for _, m := range garageHosts {
if n == m {
out = append(out, report.Fetch{ID: "garage", At: at, Status: 200, Body: "Garage is fully operational\nConsult the full health check API endpoint at /v1/health for more details\n"})
}
}
for k, m := range etcdHosts {
if n == m {
// About 20 WAL syncs a second, most well under 16 ms.
c := now.Unix()/5 + int64(k)*1000
leader := 0
if m == pgLeader {
leader = 1
}
out = append(out, report.Fetch{ID: "etcd", At: at, Status: 200, Body: fmt.Sprintf(`etcd_server_has_leader 1
etcd_server_is_leader %d
etcd_server_leader_changes_seen_total 3
etcd_mvcc_db_total_size_in_bytes %g
etcd_server_quota_backend_bytes 2.147483648e+09
etcd_disk_wal_fsync_duration_seconds_bucket{le="0.016"} %d
etcd_disk_wal_fsync_duration_seconds_bucket{le="0.064"} %d
etcd_disk_wal_fsync_duration_seconds_bucket{le="0.512"} %d
etcd_disk_wal_fsync_duration_seconds_bucket{le="+Inf"} %d
`, leader, 3.1e7+float64(k)*1e5, 90*c, 998*c/10, 100*c, 100*c)})
}
}
return out
}