From 109f1bc7803003e7d96cb3f344a54316b046593b Mon Sep 17 00:00:00 2001 From: John Coffey Date: Wed, 11 Nov 2026 23:12:23 -0700 Subject: [PATCH] cluster: a doctor that checks a node can reach the data layer `inbuxa cluster doctor` probes the endpoints a clustered inbuxa server depends on -- the PostgreSQL store, the NATS coordinator, etcd, Patroni and the S3 blob store -- and reports which answer. It checks only the endpoints named on the command line, each with that service's own probe (a NATS greeting, etcd's /health, Patroni's /primary at 200 for the leader), so it fits any deployment and touches nothing on the node. It's the reachability preflight a node wants before it is pointed at a cluster, and the groundwork for guided cluster membership. --- cmd/inbuxa/main.go | 87 +++++++++++++++++++ internal/cluster/doctor.go | 143 ++++++++++++++++++++++++++++++++ internal/cluster/doctor_test.go | 122 +++++++++++++++++++++++++++ 3 files changed, 352 insertions(+) create mode 100644 internal/cluster/doctor.go create mode 100644 internal/cluster/doctor_test.go diff --git a/cmd/inbuxa/main.go b/cmd/inbuxa/main.go index 29eb112..5fc3f19 100644 --- a/cmd/inbuxa/main.go +++ b/cmd/inbuxa/main.go @@ -21,9 +21,11 @@ import ( "path/filepath" "runtime" "strings" + "text/tabwriter" "time" "git.coffeylabs.org/inbuxa/inbuxa-installer/internal/apply" + "git.coffeylabs.org/inbuxa/inbuxa-installer/internal/cluster" "git.coffeylabs.org/inbuxa/inbuxa-installer/internal/deps" "git.coffeylabs.org/inbuxa/inbuxa-installer/internal/discover" "git.coffeylabs.org/inbuxa/inbuxa-installer/internal/host" @@ -62,6 +64,9 @@ const usage = `inbuxa -- install the inbuxa suite on this machine inbuxa relay install|status|uninstall inbuxa Relay, on a machine of its own in front of the server + inbuxa cluster doctor [flags] + whether this node can reach a cluster's data + layer (PostgreSQL, NATS, etcd, Patroni, S3) inbuxa version this program's version install flags: @@ -130,6 +135,15 @@ relay uninstall flags: --discard-held with --purge, even while it holds mail --dry-run, --yes as for install +cluster doctor flags (each endpoint is HOST:PORT; only the ones given are +checked, so it fits any deployment; omit them all and it does nothing): + --pg HOST:PORT the PostgreSQL store (often an HAProxy listener) + --nats HOST:PORT the NATS coordinator + --etcd HOST:PORT etcd, checked at http://HOST:PORT/health + --patroni HOST:PORT Patroni, checked at /primary (200 = the leader) + --s3 HOST:PORT the S3 blob store, checked at /health + --timeout DURATION how long each probe waits; default 5s + deps flags: --server/--console/--webmail SHAPE the shapes to work out the needs for --install do it, rather than only saying what it would do @@ -178,6 +192,8 @@ func main() { os.Exit(certsCmd(os.Args[2:])) case "relay": os.Exit(relayCmd(os.Args[2:])) + case "cluster": + os.Exit(clusterCmd(os.Args[2:])) case "version": fmt.Println(version) case "-h", "--help", "help": @@ -1042,6 +1058,77 @@ func relayCmd(args []string) int { return 2 } +func clusterCmd(args []string) int { + if len(args) == 0 { + fmt.Fprint(os.Stderr, "inbuxa cluster doctor [flags]\n\n"+usage) + return 2 + } + switch args[0] { + case "doctor": + return clusterDoctor(args[1:]) + case "-h", "--help", "help": + fmt.Print(usage) + return 0 + } + fmt.Fprintf(os.Stderr, "unknown cluster command %q: doctor\n", args[0]) + return 2 +} + +// clusterDoctor probes the data-layer endpoints the operator names, so a node +// can be checked before it is pointed at a cluster. It touches nothing. +func clusterDoctor(args []string) int { + fs := flag.NewFlagSet("cluster doctor", flag.ContinueOnError) + fs.Usage = func() { fmt.Print(usage) } + var ( + pg = fs.String("pg", "", "") + nats = fs.String("nats", "", "") + etcd = fs.String("etcd", "", "") + patroni = fs.String("patroni", "", "") + s3 = fs.String("s3", "", "") + timeout = fs.Duration("timeout", 5*time.Second, "") + ) + if err := fs.Parse(args); err != nil { + return 2 + } + if fs.NArg() > 0 { + fmt.Fprintf(os.Stderr, "unexpected %q: cluster doctor takes only flags\n", fs.Arg(0)) + return 2 + } + + var targets []cluster.Target + add := func(addr string, t cluster.Target) { + if addr != "" { + targets = append(targets, t) + } + } + add(*pg, cluster.Target{Name: "PostgreSQL", Kind: cluster.TCP, Addr: *pg}) + add(*nats, cluster.Target{Name: "NATS", Kind: cluster.NATS, Addr: *nats}) + add(*etcd, cluster.Target{Name: "etcd", Kind: cluster.HTTP, Addr: "http://" + *etcd + "/health"}) + add(*patroni, cluster.Target{Name: "Patroni leader", Kind: cluster.HTTP, Addr: "http://" + *patroni + "/primary", HealthyStatus: 200}) + add(*s3, cluster.Target{Name: "S3 blobs", Kind: cluster.HTTP, Addr: "http://" + *s3 + "/health"}) + + if len(targets) == 0 { + fmt.Fprintln(os.Stderr, "nothing to check: name an endpoint, such as --pg HOST:PORT (inbuxa cluster doctor -h)") + return 2 + } + + checks := cluster.Doctor(context.Background(), targets, *timeout) + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + for _, c := range checks { + mark := "ok" + if !c.OK { + mark = "unreachable" + } + fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", c.Name, c.Addr, mark, c.Detail) + } + w.Flush() + if !cluster.AllOK(checks) { + fmt.Fprintln(os.Stderr, "\nsome endpoints didn't answer; a node can't join until they do") + return 1 + } + return 0 +} + func relayInstall(ctx context.Context, args []string) int { fs := flag.NewFlagSet("relay install", flag.ContinueOnError) fs.Usage = func() { fmt.Print(usage) } diff --git a/internal/cluster/doctor.go b/internal/cluster/doctor.go new file mode 100644 index 0000000..9e29fc5 --- /dev/null +++ b/internal/cluster/doctor.go @@ -0,0 +1,143 @@ +// SPDX-FileCopyrightText: 2026 Coffey Labs LLC +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package cluster checks whether a node can reach the data layer a clustered +// inbuxa server depends on: the PostgreSQL store, the NATS coordinator, the +// S3 blob store, and the etcd and Patroni that stand behind them. It probes +// only endpoints the caller names, so it fits any deployment; the probe each +// one uses follows that service's own convention, not any one cluster's +// layout. +package cluster + +import ( + "bufio" + "context" + "fmt" + "net" + "net/http" + "strings" + "time" +) + +// Kind is how an endpoint is probed. +type Kind int + +const ( + // TCP just opens a connection: the service is listening. + TCP Kind = iota + // NATS opens a connection and reads the server's INFO greeting. + NATS + // HTTP makes a GET and checks the status (HealthyStatus, or any 2xx when + // that is zero). + HTTP +) + +// Target is one endpoint to probe. +type Target struct { + Name string // what to call it in the report, e.g. "PostgreSQL" + Kind Kind + Addr string // host:port for TCP and NATS; a URL for HTTP + // HealthyStatus is the HTTP status that means healthy; 0 accepts any 2xx. + // For Patroni's /primary, 200 means this node is the leader and 503 that + // it isn't, so the caller sets it to 200. + HealthyStatus int +} + +// Check is the result of probing one target. +type Check struct { + Name string + Addr string + OK bool + Detail string +} + +// Doctor probes every target, in order, and returns a Check for each. A nil +// context uses the background context. +func Doctor(ctx context.Context, targets []Target, timeout time.Duration) []Check { + if ctx == nil { + ctx = context.Background() + } + checks := make([]Check, 0, len(targets)) + for _, t := range targets { + checks = append(checks, probe(ctx, t, timeout)) + } + return checks +} + +// AllOK reports whether every check passed. +func AllOK(checks []Check) bool { + for _, c := range checks { + if !c.OK { + return false + } + } + return true +} + +func probe(ctx context.Context, t Target, timeout time.Duration) Check { + c := Check{Name: t.Name, Addr: t.Addr} + switch t.Kind { + case NATS: + c.OK, c.Detail = natsProbe(ctx, t.Addr, timeout) + case HTTP: + c.OK, c.Detail = httpProbe(ctx, t.Addr, t.HealthyStatus, timeout) + default: + c.OK, c.Detail = tcpProbe(ctx, t.Addr, timeout) + } + return c +} + +func tcpProbe(ctx context.Context, addr string, timeout time.Duration) (bool, string) { + conn, err := dial(ctx, addr, timeout) + if err != nil { + return false, reason(err) + } + conn.Close() + return true, "listening" +} + +// natsProbe confirms the port answers like NATS: the server sends an `INFO` +// line as soon as a client connects. +func natsProbe(ctx context.Context, addr string, timeout time.Duration) (bool, string) { + conn, err := dial(ctx, addr, timeout) + if err != nil { + return false, reason(err) + } + defer conn.Close() + _ = conn.SetReadDeadline(time.Now().Add(timeout)) + line, err := bufio.NewReader(conn).ReadString('\n') + if err != nil { + return false, "connected, but no greeting: " + reason(err) + } + if strings.HasPrefix(strings.TrimSpace(line), "INFO") { + return true, "NATS greeting" + } + return false, "connected, but the greeting wasn't NATS" +} + +func httpProbe(ctx context.Context, url string, want int, timeout time.Duration) (bool, string) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return false, reason(err) + } + resp, err := (&http.Client{Timeout: timeout}).Do(req) + if err != nil { + return false, reason(err) + } + defer resp.Body.Close() + ok := resp.StatusCode == want || (want == 0 && resp.StatusCode >= 200 && resp.StatusCode < 300) + return ok, fmt.Sprintf("HTTP %d", resp.StatusCode) +} + +func dial(ctx context.Context, addr string, timeout time.Duration) (net.Conn, error) { + return (&net.Dialer{Timeout: timeout}).DialContext(ctx, "tcp", addr) +} + +// reason trims a dial error to the part worth showing. +func reason(err error) string { + msg := err.Error() + if i := strings.LastIndex(msg, ": "); i >= 0 { + return msg[i+2:] + } + return msg +} diff --git a/internal/cluster/doctor_test.go b/internal/cluster/doctor_test.go new file mode 100644 index 0000000..1fe61f8 --- /dev/null +++ b/internal/cluster/doctor_test.go @@ -0,0 +1,122 @@ +// SPDX-FileCopyrightText: 2026 Coffey Labs LLC +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cluster + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +const wait = 2 * time.Second + +// listenTCP opens a throwaway TCP listener and returns its address. If greet +// is non-empty, it's written to the first connection, like a NATS greeting. +func listenTCP(t *testing.T, greet string) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { ln.Close() }) + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + if greet != "" { + conn.Write([]byte(greet)) + } + conn.Close() + } + }() + return ln.Addr().String() +} + +// closedAddr returns an address nothing listens on. +func closedAddr(t *testing.T) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + addr := ln.Addr().String() + ln.Close() + return addr +} + +func TestTCPProbe(t *testing.T) { + open := listenTCP(t, "") + if ok, _ := tcpProbe(context.Background(), open, wait); !ok { + t.Error("a listening port should probe OK") + } + if ok, detail := tcpProbe(context.Background(), closedAddr(t), wait); ok { + t.Errorf("a closed port should fail, got OK (%s)", detail) + } +} + +func TestNATSProbe(t *testing.T) { + nats := listenTCP(t, "INFO {\"server_id\":\"x\"}\r\n") + if ok, detail := natsProbe(context.Background(), nats, wait); !ok { + t.Errorf("a NATS greeting should probe OK, got %q", detail) + } + // A port that's open but doesn't greet like NATS fails. + notNats := listenTCP(t, "+OK\r\n") + if ok, _ := natsProbe(context.Background(), notNats, wait); ok { + t.Error("a non-NATS greeting should fail") + } + if ok, _ := natsProbe(context.Background(), closedAddr(t), wait); ok { + t.Error("a closed port should fail") + } +} + +func TestHTTPProbe(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/health": + w.WriteHeader(http.StatusOK) + case "/primary": + w.WriteHeader(http.StatusServiceUnavailable) // not the leader + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + + // Any 2xx when the wanted status is 0. + if ok, detail := httpProbe(context.Background(), srv.URL+"/health", 0, wait); !ok { + t.Errorf("/health 200 should pass, got %q", detail) + } + // Patroni: 200 means leader; 503 (not leader) must fail when 200 is wanted. + if ok, _ := httpProbe(context.Background(), srv.URL+"/primary", 200, wait); ok { + t.Error("/primary 503 should fail when 200 is wanted") + } + if ok, _ := httpProbe(context.Background(), srv.URL+"/missing", 0, wait); ok { + t.Error("404 should fail") + } +} + +func TestDoctorAndAllOK(t *testing.T) { + open := listenTCP(t, "") + checks := Doctor(nil, []Target{ + {Name: "PostgreSQL", Kind: TCP, Addr: open}, + {Name: "NATS", Kind: TCP, Addr: closedAddr(t)}, + }, wait) + if len(checks) != 2 { + t.Fatalf("want 2 checks, got %d", len(checks)) + } + if !checks[0].OK || checks[1].OK { + t.Errorf("unexpected results: %+v", checks) + } + if AllOK(checks) { + t.Error("AllOK should be false when one check failed") + } + if !AllOK(checks[:1]) { + t.Error("AllOK should be true when every check passed") + } +} -- 2.54.0