cluster: a doctor that checks a node can reach the data layer #53

Merged
jcoffey-dev merged 1 commits from feat/cluster-doctor into main 2026-11-12 14:02:11 +00:00
3 changed files with 352 additions and 0 deletions

No files matched your search

+87
View File
@@ -21,9 +21,11 @@ import (
"path/filepath"
"runtime"
"strings"
"text/tabwriter"
"time"
"git.coffeylabs.org/inbuxa/inbuxa-installer/internal/apply"
"git.coffeylabs.org/inbuxa/inbuxa-installer/internal/cluster"
"git.coffeylabs.org/inbuxa/inbuxa-installer/internal/deps"
"git.coffeylabs.org/inbuxa/inbuxa-installer/internal/discover"
"git.coffeylabs.org/inbuxa/inbuxa-installer/internal/host"
@@ -62,6 +64,9 @@ const usage = `inbuxa -- install the inbuxa suite on this machine
inbuxa relay install|status|uninstall
inbuxa Relay, on a machine of its own in front
of the server
inbuxa cluster doctor [flags]
whether this node can reach a cluster's data
layer (PostgreSQL, NATS, etcd, Patroni, S3)
inbuxa version this program's version
install flags:
@@ -130,6 +135,15 @@ relay uninstall flags:
--discard-held with --purge, even while it holds mail
--dry-run, --yes as for install
cluster doctor flags (each endpoint is HOST:PORT; only the ones given are
checked, so it fits any deployment; omit them all and it does nothing):
--pg HOST:PORT the PostgreSQL store (often an HAProxy listener)
--nats HOST:PORT the NATS coordinator
--etcd HOST:PORT etcd, checked at http://HOST:PORT/health
--patroni HOST:PORT Patroni, checked at /primary (200 = the leader)
--s3 HOST:PORT the S3 blob store, checked at /health
--timeout DURATION how long each probe waits; default 5s
deps flags:
--server/--console/--webmail SHAPE the shapes to work out the needs for
--install do it, rather than only saying what it would do
@@ -178,6 +192,8 @@ func main() {
os.Exit(certsCmd(os.Args[2:]))
case "relay":
os.Exit(relayCmd(os.Args[2:]))
case "cluster":
os.Exit(clusterCmd(os.Args[2:]))
case "version":
fmt.Println(version)
case "-h", "--help", "help":
@@ -1042,6 +1058,77 @@ func relayCmd(args []string) int {
return 2
}
func clusterCmd(args []string) int {
if len(args) == 0 {
fmt.Fprint(os.Stderr, "inbuxa cluster doctor [flags]\n\n"+usage)
return 2
}
switch args[0] {
case "doctor":
return clusterDoctor(args[1:])
case "-h", "--help", "help":
fmt.Print(usage)
return 0
}
fmt.Fprintf(os.Stderr, "unknown cluster command %q: doctor\n", args[0])
return 2
}
// clusterDoctor probes the data-layer endpoints the operator names, so a node
// can be checked before it is pointed at a cluster. It touches nothing.
func clusterDoctor(args []string) int {
fs := flag.NewFlagSet("cluster doctor", flag.ContinueOnError)
fs.Usage = func() { fmt.Print(usage) }
var (
pg = fs.String("pg", "", "")
nats = fs.String("nats", "", "")
etcd = fs.String("etcd", "", "")
patroni = fs.String("patroni", "", "")
s3 = fs.String("s3", "", "")
timeout = fs.Duration("timeout", 5*time.Second, "")
)
if err := fs.Parse(args); err != nil {
return 2
}
if fs.NArg() > 0 {
fmt.Fprintf(os.Stderr, "unexpected %q: cluster doctor takes only flags\n", fs.Arg(0))
return 2
}
var targets []cluster.Target
add := func(addr string, t cluster.Target) {
if addr != "" {
targets = append(targets, t)
}
}
add(*pg, cluster.Target{Name: "PostgreSQL", Kind: cluster.TCP, Addr: *pg})
add(*nats, cluster.Target{Name: "NATS", Kind: cluster.NATS, Addr: *nats})
add(*etcd, cluster.Target{Name: "etcd", Kind: cluster.HTTP, Addr: "http://" + *etcd + "/health"})
add(*patroni, cluster.Target{Name: "Patroni leader", Kind: cluster.HTTP, Addr: "http://" + *patroni + "/primary", HealthyStatus: 200})
add(*s3, cluster.Target{Name: "S3 blobs", Kind: cluster.HTTP, Addr: "http://" + *s3 + "/health"})
if len(targets) == 0 {
fmt.Fprintln(os.Stderr, "nothing to check: name an endpoint, such as --pg HOST:PORT (inbuxa cluster doctor -h)")
return 2
}
checks := cluster.Doctor(context.Background(), targets, *timeout)
w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
for _, c := range checks {
mark := "ok"
if !c.OK {
mark = "unreachable"
}
fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", c.Name, c.Addr, mark, c.Detail)
}
w.Flush()
if !cluster.AllOK(checks) {
fmt.Fprintln(os.Stderr, "\nsome endpoints didn't answer; a node can't join until they do")
return 1
}
return 0
}
func relayInstall(ctx context.Context, args []string) int {
fs := flag.NewFlagSet("relay install", flag.ContinueOnError)
fs.Usage = func() { fmt.Print(usage) }
+143
View File
@@ -0,0 +1,143 @@
// SPDX-FileCopyrightText: 2026 Coffey Labs LLC
// SPDX-License-Identifier: AGPL-3.0-or-later
// Package cluster checks whether a node can reach the data layer a clustered
// inbuxa server depends on: the PostgreSQL store, the NATS coordinator, the
// S3 blob store, and the etcd and Patroni that stand behind them. It probes
// only endpoints the caller names, so it fits any deployment; the probe each
// one uses follows that service's own convention, not any one cluster's
// layout.
package cluster
import (
"bufio"
"context"
"fmt"
"net"
"net/http"
"strings"
"time"
)
// Kind is how an endpoint is probed.
type Kind int
const (
// TCP just opens a connection: the service is listening.
TCP Kind = iota
// NATS opens a connection and reads the server's INFO greeting.
NATS
// HTTP makes a GET and checks the status (HealthyStatus, or any 2xx when
// that is zero).
HTTP
)
// Target is one endpoint to probe.
type Target struct {
Name string // what to call it in the report, e.g. "PostgreSQL"
Kind Kind
Addr string // host:port for TCP and NATS; a URL for HTTP
// HealthyStatus is the HTTP status that means healthy; 0 accepts any 2xx.
// For Patroni's /primary, 200 means this node is the leader and 503 that
// it isn't, so the caller sets it to 200.
HealthyStatus int
}
// Check is the result of probing one target.
type Check struct {
Name string
Addr string
OK bool
Detail string
}
// Doctor probes every target, in order, and returns a Check for each. A nil
// context uses the background context.
func Doctor(ctx context.Context, targets []Target, timeout time.Duration) []Check {
if ctx == nil {
ctx = context.Background()
}
checks := make([]Check, 0, len(targets))
for _, t := range targets {
checks = append(checks, probe(ctx, t, timeout))
}
return checks
}
// AllOK reports whether every check passed.
func AllOK(checks []Check) bool {
for _, c := range checks {
if !c.OK {
return false
}
}
return true
}
func probe(ctx context.Context, t Target, timeout time.Duration) Check {
c := Check{Name: t.Name, Addr: t.Addr}
switch t.Kind {
case NATS:
c.OK, c.Detail = natsProbe(ctx, t.Addr, timeout)
case HTTP:
c.OK, c.Detail = httpProbe(ctx, t.Addr, t.HealthyStatus, timeout)
default:
c.OK, c.Detail = tcpProbe(ctx, t.Addr, timeout)
}
return c
}
func tcpProbe(ctx context.Context, addr string, timeout time.Duration) (bool, string) {
conn, err := dial(ctx, addr, timeout)
if err != nil {
return false, reason(err)
}
conn.Close()
return true, "listening"
}
// natsProbe confirms the port answers like NATS: the server sends an `INFO`
// line as soon as a client connects.
func natsProbe(ctx context.Context, addr string, timeout time.Duration) (bool, string) {
conn, err := dial(ctx, addr, timeout)
if err != nil {
return false, reason(err)
}
defer conn.Close()
_ = conn.SetReadDeadline(time.Now().Add(timeout))
line, err := bufio.NewReader(conn).ReadString('\n')
if err != nil {
return false, "connected, but no greeting: " + reason(err)
}
if strings.HasPrefix(strings.TrimSpace(line), "INFO") {
return true, "NATS greeting"
}
return false, "connected, but the greeting wasn't NATS"
}
func httpProbe(ctx context.Context, url string, want int, timeout time.Duration) (bool, string) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return false, reason(err)
}
resp, err := (&http.Client{Timeout: timeout}).Do(req)
if err != nil {
return false, reason(err)
}
defer resp.Body.Close()
ok := resp.StatusCode == want || (want == 0 && resp.StatusCode >= 200 && resp.StatusCode < 300)
return ok, fmt.Sprintf("HTTP %d", resp.StatusCode)
}
func dial(ctx context.Context, addr string, timeout time.Duration) (net.Conn, error) {
return (&net.Dialer{Timeout: timeout}).DialContext(ctx, "tcp", addr)
}
// reason trims a dial error to the part worth showing.
func reason(err error) string {
msg := err.Error()
if i := strings.LastIndex(msg, ": "); i >= 0 {
return msg[i+2:]
}
return msg
}
+122
View File
@@ -0,0 +1,122 @@
// SPDX-FileCopyrightText: 2026 Coffey Labs LLC
// SPDX-License-Identifier: AGPL-3.0-or-later
package cluster
import (
"context"
"net"
"net/http"
"net/http/httptest"
"testing"
"time"
)
const wait = 2 * time.Second
// listenTCP opens a throwaway TCP listener and returns its address. If greet
// is non-empty, it's written to the first connection, like a NATS greeting.
func listenTCP(t *testing.T, greet string) string {
t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { ln.Close() })
go func() {
for {
conn, err := ln.Accept()
if err != nil {
return
}
if greet != "" {
conn.Write([]byte(greet))
}
conn.Close()
}
}()
return ln.Addr().String()
}
// closedAddr returns an address nothing listens on.
func closedAddr(t *testing.T) string {
t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
addr := ln.Addr().String()
ln.Close()
return addr
}
func TestTCPProbe(t *testing.T) {
open := listenTCP(t, "")
if ok, _ := tcpProbe(context.Background(), open, wait); !ok {
t.Error("a listening port should probe OK")
}
if ok, detail := tcpProbe(context.Background(), closedAddr(t), wait); ok {
t.Errorf("a closed port should fail, got OK (%s)", detail)
}
}
func TestNATSProbe(t *testing.T) {
nats := listenTCP(t, "INFO {\"server_id\":\"x\"}\r\n")
if ok, detail := natsProbe(context.Background(), nats, wait); !ok {
t.Errorf("a NATS greeting should probe OK, got %q", detail)
}
// A port that's open but doesn't greet like NATS fails.
notNats := listenTCP(t, "+OK\r\n")
if ok, _ := natsProbe(context.Background(), notNats, wait); ok {
t.Error("a non-NATS greeting should fail")
}
if ok, _ := natsProbe(context.Background(), closedAddr(t), wait); ok {
t.Error("a closed port should fail")
}
}
func TestHTTPProbe(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/health":
w.WriteHeader(http.StatusOK)
case "/primary":
w.WriteHeader(http.StatusServiceUnavailable) // not the leader
default:
w.WriteHeader(http.StatusNotFound)
}
}))
t.Cleanup(srv.Close)
// Any 2xx when the wanted status is 0.
if ok, detail := httpProbe(context.Background(), srv.URL+"/health", 0, wait); !ok {
t.Errorf("/health 200 should pass, got %q", detail)
}
// Patroni: 200 means leader; 503 (not leader) must fail when 200 is wanted.
if ok, _ := httpProbe(context.Background(), srv.URL+"/primary", 200, wait); ok {
t.Error("/primary 503 should fail when 200 is wanted")
}
if ok, _ := httpProbe(context.Background(), srv.URL+"/missing", 0, wait); ok {
t.Error("404 should fail")
}
}
func TestDoctorAndAllOK(t *testing.T) {
open := listenTCP(t, "")
checks := Doctor(nil, []Target{
{Name: "PostgreSQL", Kind: TCP, Addr: open},
{Name: "NATS", Kind: TCP, Addr: closedAddr(t)},
}, wait)
if len(checks) != 2 {
t.Fatalf("want 2 checks, got %d", len(checks))
}
if !checks[0].OK || checks[1].OK {
t.Errorf("unexpected results: %+v", checks)
}
if AllOK(checks) {
t.Error("AllOK should be false when one check failed")
}
if !AllOK(checks[:1]) {
t.Error("AllOK should be true when every check passed")
}
}