Files
cairnobs/scripts/fleet/datalayer.go
T
jcoffey-dev 7b2df94812 fleet: the demo's data layer; etcd's disk sync figure holds between reports
With -demo and enough hosts, site 0 gets a Patroni cluster (fleet serves
each member's API on its own loopback address), HAProxy in front of it,
a three-node NATS cluster, Garage and etcd, read as an agent's fetches
are. The leader is away from its preferred member and one replica is
40 MB behind, so the Data layer page and alerts have something to show.
config -demo adds the datalayer: section to demo.yaml.

Found on the way:
- etcd's disk sync p99 was worked out from the reading before, and every
  look (the page and the evaluator, several times per report) replaced
  that reading, so it rarely had new syncs to go by and mostly showed
  "measuring...". It now moves on only when a report brings 5 or more new
  syncs, and keeps the last figure in between.
- The Data layer page's intro spoke of "the mail cluster", and the leader
  warning named our own switchover tool: both now say what any install
  has (patronictl switchover).
- An HAProxy backend with many servers listed them all and pushed the
  table sideways; past three it says how many are up, with the names on
  hover.
2026-11-14 20:00:04 -07:00

176 lines
6.0 KiB
Go

package main
import (
"context"
"fmt"
"log"
"net"
"net/http"
"strings"
"time"
"github.com/cairnobs/cairnobs/internal/report"
)
// The demo's data layer (Data layer page, docs/formats.md): site 0's first
// three databases are a Patroni cluster, its load balancers run HAProxy in
// front of it, its NATS hosts a three-node cluster, and its caches a Garage
// cluster. Each Patroni member answers on its own loopback address
// (pgAddr), so obs-beacond can match a member to its API as it does over a
// real network; the status pages come in the hosts' reports, as an agent's
// fetches do.
// Problems on purpose: the leader is fleet-0031, not its usual fleet-0030
// (a failover nobody moved back), and fleet-0032 is 40 MB behind.
const patroniPort = 58420
var (
pgMembers = []int{30, 31, 32}
pgLeader = 31
pgHome = 30
pgLagging = 32
natsHosts = []int{45, 46, 47}
garageHosts = []int{40, 41, 42}
etcdHosts = pgMembers
haproxyLBs = []int{1, 2}
)
// hasDataLayer: the fleet has every host the data layer names.
func hasDataLayer(hosts int) bool { return hosts > 47 }
// dataLayerYAML is the datalayer: section for obs-beacond.yaml.
func dataLayerYAML() string {
names := func(ns []int) string {
var s []string
for _, n := range ns {
s = append(s, hostName(n))
}
return strings.Join(s, ", ")
}
var urls []string
for _, m := range pgMembers {
urls = append(urls, fmt.Sprintf("http://%s:%d", pgAddr(m), patroniPort))
}
return fmt.Sprintf(`datalayer:
patroni: [%s]
sync_standby: true
preferred_leader: %s
probes:
- {id: haproxy, kind: haproxy, url: "http://127.0.0.1:8404/stats;csv", from: [%s], exactly_one: [pg-primary], at_least_one: [pg-replicas]}
- {id: nats, kind: nats, url: "http://127.0.0.1:8222/varz", from: [%s], peers: 2}
- {id: garage, kind: garage, url: "http://127.0.0.1:3903/health", from: [%s]}
- {id: etcd, kind: etcd, url: "http://127.0.0.1:2381/metrics", from: [%s], fsync_warn_ms: 500}
`, strings.Join(urls, ", "), hostName(pgHome), names(haproxyLBs), names(natsHosts), names(garageHosts), names(etcdHosts))
}
// pgAddr is member n's loopback address.
func pgAddr(n int) string { return fmt.Sprintf("127.0.84.%d", n) }
// servePatroni answers /cluster and /patroni for each member.
func servePatroni(ctx context.Context) {
for _, me := range pgMembers {
ln, err := net.Listen("tcp", fmt.Sprintf("%s:%d", pgAddr(me), patroniPort))
if err != nil {
log.Fatal(err)
}
srv := &http.Server{ReadHeaderTimeout: 5 * time.Second, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch r.URL.Path {
case "/cluster":
var ms []string
for _, m := range pgMembers {
role, state, lag := "replica", "streaming", `,"lag":0`
switch m {
case pgLeader:
role, state, lag = "leader", "running", ""
case pgHome:
role = "sync_standby"
case pgLagging:
lag = `,"lag":41943040`
}
ms = append(ms, fmt.Sprintf(`{"name":%q,"role":%q,"state":%q,"host":%q,"port":5432,"timeline":7%s}`, hostName(m), role, state, pgAddr(m), lag))
}
fmt.Fprintf(w, `{"members":[%s],"scope":"shop-pg"}`, strings.Join(ms, ","))
case "/patroni":
role := "replica"
if me == pgLeader {
role = "primary"
}
fmt.Fprintf(w, `{"state":"running","role":%q,"dcs_last_seen":%d,"timeline":7}`, role, time.Now().Unix()-3)
default:
http.NotFound(w, r)
}
})}
go func() { _ = srv.Serve(ln) }()
go func() { <-ctx.Done(); _ = srv.Close() }()
}
log.Printf("fleet: Patroni cluster shop-pg on %s-%d port %d", pgAddr(pgMembers[0]), pgMembers[len(pgMembers)-1], patroniPort)
}
// dataFetches are the status pages host n reads, if any (site 0 only).
func dataFetches(n int, now time.Time) []report.Fetch {
if n >= siteSize {
return nil
}
at := now.Add(-2 * time.Second).UTC()
var out []report.Fetch
for _, lb := range haproxyLBs {
if n != lb {
continue
}
var b strings.Builder
b.WriteString("# pxname,svname,qcur,status,weight\nstats,FRONTEND,,OPEN,\n")
backend := func(name string, up func(m int) bool, servers []int) {
fmt.Fprintf(&b, "%s,FRONTEND,,OPEN,\n", name)
for _, m := range servers {
st := "DOWN"
if up(m) {
st = "UP"
}
fmt.Fprintf(&b, "%s,%s,0,%s,1\n", name, hostName(m), st)
}
fmt.Fprintf(&b, "%s,BACKEND,0,UP,1\n", name)
}
backend("pg-primary", func(m int) bool { return m == pgLeader }, pgMembers)
backend("pg-replicas", func(m int) bool { return m != pgLeader }, pgMembers)
var apps []int
for p := 3; p < 13; p++ {
apps = append(apps, p)
}
backend("shop-web", func(int) bool { return true }, apps)
out = append(out, report.Fetch{ID: "haproxy", At: at, Status: 200, Body: b.String()})
}
for k, m := range natsHosts {
if n == m {
out = append(out, report.Fetch{ID: "nats", At: at, Status: 200, Body: fmt.Sprintf(
`{"server_name":%q,"version":"2.11.17","connections":%d,"routes":2,"remotes":2,"slow_consumers":0,"in_msgs":%d,"out_msgs":%d,"uptime":"21d4h"}`,
hostName(m), 40+k*3, 1_900_000+k*1000, 2_400_000+k*1000)})
}
}
for _, m := range garageHosts {
if n == m {
out = append(out, report.Fetch{ID: "garage", At: at, Status: 200, Body: "Garage is fully operational\nConsult the full health check API endpoint at /v1/health for more details\n"})
}
}
for k, m := range etcdHosts {
if n == m {
// About 20 WAL syncs a second, most well under 16 ms.
c := now.Unix()/5 + int64(k)*1000
leader := 0
if m == pgLeader {
leader = 1
}
out = append(out, report.Fetch{ID: "etcd", At: at, Status: 200, Body: fmt.Sprintf(`etcd_server_has_leader 1
etcd_server_is_leader %d
etcd_server_leader_changes_seen_total 3
etcd_mvcc_db_total_size_in_bytes %g
etcd_server_quota_backend_bytes 2.147483648e+09
etcd_disk_wal_fsync_duration_seconds_bucket{le="0.016"} %d
etcd_disk_wal_fsync_duration_seconds_bucket{le="0.064"} %d
etcd_disk_wal_fsync_duration_seconds_bucket{le="0.512"} %d
etcd_disk_wal_fsync_duration_seconds_bucket{le="+Inf"} %d
`, leader, 3.1e7+float64(k)*1e5, 90*c, 998*c/10, 100*c, 100*c)})
}
}
return out
}