With -demo and enough hosts, site 0 gets a Patroni cluster (fleet serves each member's API on its own loopback address), HAProxy in front of it, a three-node NATS cluster, Garage and etcd, read as an agent's fetches are. The leader is away from its preferred member and one replica is 40 MB behind, so the Data layer page and alerts have something to show. config -demo adds the datalayer: section to demo.yaml. Found on the way: - etcd's disk sync p99 was worked out from the reading before, and every look (the page and the evaluator, several times per report) replaced that reading, so it rarely had new syncs to go by and mostly showed "measuring...". It now moves on only when a report brings 5 or more new syncs, and keeps the last figure in between. - The Data layer page's intro spoke of "the mail cluster", and the leader warning named our own switchover tool: both now say what any install has (patronictl switchover). - An HAProxy backend with many servers listed them all and pushed the table sideways; past three it says how many are up, with the names on hover.
176 lines
6.0 KiB
Go
176 lines
6.0 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log"
|
|
"net"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/cairnobs/cairnobs/internal/report"
|
|
)
|
|
|
|
// The demo's data layer (Data layer page, docs/formats.md): site 0's first
|
|
// three databases are a Patroni cluster, its load balancers run HAProxy in
|
|
// front of it, its NATS hosts a three-node cluster, and its caches a Garage
|
|
// cluster. Each Patroni member answers on its own loopback address
|
|
// (pgAddr), so obs-beacond can match a member to its API as it does over a
|
|
// real network; the status pages come in the hosts' reports, as an agent's
|
|
// fetches do.
|
|
// Problems on purpose: the leader is fleet-0031, not its usual fleet-0030
|
|
// (a failover nobody moved back), and fleet-0032 is 40 MB behind.
|
|
const patroniPort = 58420
|
|
|
|
var (
|
|
pgMembers = []int{30, 31, 32}
|
|
pgLeader = 31
|
|
pgHome = 30
|
|
pgLagging = 32
|
|
natsHosts = []int{45, 46, 47}
|
|
garageHosts = []int{40, 41, 42}
|
|
etcdHosts = pgMembers
|
|
haproxyLBs = []int{1, 2}
|
|
)
|
|
|
|
// hasDataLayer: the fleet has every host the data layer names.
|
|
func hasDataLayer(hosts int) bool { return hosts > 47 }
|
|
|
|
// dataLayerYAML is the datalayer: section for obs-beacond.yaml.
|
|
func dataLayerYAML() string {
|
|
names := func(ns []int) string {
|
|
var s []string
|
|
for _, n := range ns {
|
|
s = append(s, hostName(n))
|
|
}
|
|
return strings.Join(s, ", ")
|
|
}
|
|
var urls []string
|
|
for _, m := range pgMembers {
|
|
urls = append(urls, fmt.Sprintf("http://%s:%d", pgAddr(m), patroniPort))
|
|
}
|
|
return fmt.Sprintf(`datalayer:
|
|
patroni: [%s]
|
|
sync_standby: true
|
|
preferred_leader: %s
|
|
probes:
|
|
- {id: haproxy, kind: haproxy, url: "http://127.0.0.1:8404/stats;csv", from: [%s], exactly_one: [pg-primary], at_least_one: [pg-replicas]}
|
|
- {id: nats, kind: nats, url: "http://127.0.0.1:8222/varz", from: [%s], peers: 2}
|
|
- {id: garage, kind: garage, url: "http://127.0.0.1:3903/health", from: [%s]}
|
|
- {id: etcd, kind: etcd, url: "http://127.0.0.1:2381/metrics", from: [%s], fsync_warn_ms: 500}
|
|
`, strings.Join(urls, ", "), hostName(pgHome), names(haproxyLBs), names(natsHosts), names(garageHosts), names(etcdHosts))
|
|
}
|
|
|
|
// pgAddr is member n's loopback address.
|
|
func pgAddr(n int) string { return fmt.Sprintf("127.0.84.%d", n) }
|
|
|
|
// servePatroni answers /cluster and /patroni for each member.
|
|
func servePatroni(ctx context.Context) {
|
|
for _, me := range pgMembers {
|
|
ln, err := net.Listen("tcp", fmt.Sprintf("%s:%d", pgAddr(me), patroniPort))
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
srv := &http.Server{ReadHeaderTimeout: 5 * time.Second, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
switch r.URL.Path {
|
|
case "/cluster":
|
|
var ms []string
|
|
for _, m := range pgMembers {
|
|
role, state, lag := "replica", "streaming", `,"lag":0`
|
|
switch m {
|
|
case pgLeader:
|
|
role, state, lag = "leader", "running", ""
|
|
case pgHome:
|
|
role = "sync_standby"
|
|
case pgLagging:
|
|
lag = `,"lag":41943040`
|
|
}
|
|
ms = append(ms, fmt.Sprintf(`{"name":%q,"role":%q,"state":%q,"host":%q,"port":5432,"timeline":7%s}`, hostName(m), role, state, pgAddr(m), lag))
|
|
}
|
|
fmt.Fprintf(w, `{"members":[%s],"scope":"shop-pg"}`, strings.Join(ms, ","))
|
|
case "/patroni":
|
|
role := "replica"
|
|
if me == pgLeader {
|
|
role = "primary"
|
|
}
|
|
fmt.Fprintf(w, `{"state":"running","role":%q,"dcs_last_seen":%d,"timeline":7}`, role, time.Now().Unix()-3)
|
|
default:
|
|
http.NotFound(w, r)
|
|
}
|
|
})}
|
|
go func() { _ = srv.Serve(ln) }()
|
|
go func() { <-ctx.Done(); _ = srv.Close() }()
|
|
}
|
|
log.Printf("fleet: Patroni cluster shop-pg on %s-%d port %d", pgAddr(pgMembers[0]), pgMembers[len(pgMembers)-1], patroniPort)
|
|
}
|
|
|
|
// dataFetches are the status pages host n reads, if any (site 0 only).
|
|
func dataFetches(n int, now time.Time) []report.Fetch {
|
|
if n >= siteSize {
|
|
return nil
|
|
}
|
|
at := now.Add(-2 * time.Second).UTC()
|
|
var out []report.Fetch
|
|
for _, lb := range haproxyLBs {
|
|
if n != lb {
|
|
continue
|
|
}
|
|
var b strings.Builder
|
|
b.WriteString("# pxname,svname,qcur,status,weight\nstats,FRONTEND,,OPEN,\n")
|
|
backend := func(name string, up func(m int) bool, servers []int) {
|
|
fmt.Fprintf(&b, "%s,FRONTEND,,OPEN,\n", name)
|
|
for _, m := range servers {
|
|
st := "DOWN"
|
|
if up(m) {
|
|
st = "UP"
|
|
}
|
|
fmt.Fprintf(&b, "%s,%s,0,%s,1\n", name, hostName(m), st)
|
|
}
|
|
fmt.Fprintf(&b, "%s,BACKEND,0,UP,1\n", name)
|
|
}
|
|
backend("pg-primary", func(m int) bool { return m == pgLeader }, pgMembers)
|
|
backend("pg-replicas", func(m int) bool { return m != pgLeader }, pgMembers)
|
|
var apps []int
|
|
for p := 3; p < 13; p++ {
|
|
apps = append(apps, p)
|
|
}
|
|
backend("shop-web", func(int) bool { return true }, apps)
|
|
out = append(out, report.Fetch{ID: "haproxy", At: at, Status: 200, Body: b.String()})
|
|
}
|
|
for k, m := range natsHosts {
|
|
if n == m {
|
|
out = append(out, report.Fetch{ID: "nats", At: at, Status: 200, Body: fmt.Sprintf(
|
|
`{"server_name":%q,"version":"2.11.17","connections":%d,"routes":2,"remotes":2,"slow_consumers":0,"in_msgs":%d,"out_msgs":%d,"uptime":"21d4h"}`,
|
|
hostName(m), 40+k*3, 1_900_000+k*1000, 2_400_000+k*1000)})
|
|
}
|
|
}
|
|
for _, m := range garageHosts {
|
|
if n == m {
|
|
out = append(out, report.Fetch{ID: "garage", At: at, Status: 200, Body: "Garage is fully operational\nConsult the full health check API endpoint at /v1/health for more details\n"})
|
|
}
|
|
}
|
|
for k, m := range etcdHosts {
|
|
if n == m {
|
|
// About 20 WAL syncs a second, most well under 16 ms.
|
|
c := now.Unix()/5 + int64(k)*1000
|
|
leader := 0
|
|
if m == pgLeader {
|
|
leader = 1
|
|
}
|
|
out = append(out, report.Fetch{ID: "etcd", At: at, Status: 200, Body: fmt.Sprintf(`etcd_server_has_leader 1
|
|
etcd_server_is_leader %d
|
|
etcd_server_leader_changes_seen_total 3
|
|
etcd_mvcc_db_total_size_in_bytes %g
|
|
etcd_server_quota_backend_bytes 2.147483648e+09
|
|
etcd_disk_wal_fsync_duration_seconds_bucket{le="0.016"} %d
|
|
etcd_disk_wal_fsync_duration_seconds_bucket{le="0.064"} %d
|
|
etcd_disk_wal_fsync_duration_seconds_bucket{le="0.512"} %d
|
|
etcd_disk_wal_fsync_duration_seconds_bucket{le="+Inf"} %d
|
|
`, leader, 3.1e7+float64(k)*1e5, 90*c, 998*c/10, 100*c, 100*c)})
|
|
}
|
|
}
|
|
return out
|
|
}
|