cluster: a doctor that checks a node can reach the data layer #53
No files matched your search
@@ -21,9 +21,11 @@ import (
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
|
||||
"git.coffeylabs.org/inbuxa/inbuxa-installer/internal/apply"
|
||||
"git.coffeylabs.org/inbuxa/inbuxa-installer/internal/cluster"
|
||||
"git.coffeylabs.org/inbuxa/inbuxa-installer/internal/deps"
|
||||
"git.coffeylabs.org/inbuxa/inbuxa-installer/internal/discover"
|
||||
"git.coffeylabs.org/inbuxa/inbuxa-installer/internal/host"
|
||||
@@ -62,6 +64,9 @@ const usage = `inbuxa -- install the inbuxa suite on this machine
|
||||
inbuxa relay install|status|uninstall
|
||||
inbuxa Relay, on a machine of its own in front
|
||||
of the server
|
||||
inbuxa cluster doctor [flags]
|
||||
whether this node can reach a cluster's data
|
||||
layer (PostgreSQL, NATS, etcd, Patroni, S3)
|
||||
inbuxa version this program's version
|
||||
|
||||
install flags:
|
||||
@@ -130,6 +135,15 @@ relay uninstall flags:
|
||||
--discard-held with --purge, even while it holds mail
|
||||
--dry-run, --yes as for install
|
||||
|
||||
cluster doctor flags (each endpoint is HOST:PORT; only the ones given are
|
||||
checked, so it fits any deployment; omit them all and it does nothing):
|
||||
--pg HOST:PORT the PostgreSQL store (often an HAProxy listener)
|
||||
--nats HOST:PORT the NATS coordinator
|
||||
--etcd HOST:PORT etcd, checked at http://HOST:PORT/health
|
||||
--patroni HOST:PORT Patroni, checked at /primary (200 = the leader)
|
||||
--s3 HOST:PORT the S3 blob store, checked at /health
|
||||
--timeout DURATION how long each probe waits; default 5s
|
||||
|
||||
deps flags:
|
||||
--server/--console/--webmail SHAPE the shapes to work out the needs for
|
||||
--install do it, rather than only saying what it would do
|
||||
@@ -178,6 +192,8 @@ func main() {
|
||||
os.Exit(certsCmd(os.Args[2:]))
|
||||
case "relay":
|
||||
os.Exit(relayCmd(os.Args[2:]))
|
||||
case "cluster":
|
||||
os.Exit(clusterCmd(os.Args[2:]))
|
||||
case "version":
|
||||
fmt.Println(version)
|
||||
case "-h", "--help", "help":
|
||||
@@ -1042,6 +1058,77 @@ func relayCmd(args []string) int {
|
||||
return 2
|
||||
}
|
||||
|
||||
func clusterCmd(args []string) int {
|
||||
if len(args) == 0 {
|
||||
fmt.Fprint(os.Stderr, "inbuxa cluster doctor [flags]\n\n"+usage)
|
||||
return 2
|
||||
}
|
||||
switch args[0] {
|
||||
case "doctor":
|
||||
return clusterDoctor(args[1:])
|
||||
case "-h", "--help", "help":
|
||||
fmt.Print(usage)
|
||||
return 0
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "unknown cluster command %q: doctor\n", args[0])
|
||||
return 2
|
||||
}
|
||||
|
||||
// clusterDoctor probes the data-layer endpoints the operator names, so a node
|
||||
// can be checked before it is pointed at a cluster. It touches nothing.
|
||||
func clusterDoctor(args []string) int {
|
||||
fs := flag.NewFlagSet("cluster doctor", flag.ContinueOnError)
|
||||
fs.Usage = func() { fmt.Print(usage) }
|
||||
var (
|
||||
pg = fs.String("pg", "", "")
|
||||
nats = fs.String("nats", "", "")
|
||||
etcd = fs.String("etcd", "", "")
|
||||
patroni = fs.String("patroni", "", "")
|
||||
s3 = fs.String("s3", "", "")
|
||||
timeout = fs.Duration("timeout", 5*time.Second, "")
|
||||
)
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return 2
|
||||
}
|
||||
if fs.NArg() > 0 {
|
||||
fmt.Fprintf(os.Stderr, "unexpected %q: cluster doctor takes only flags\n", fs.Arg(0))
|
||||
return 2
|
||||
}
|
||||
|
||||
var targets []cluster.Target
|
||||
add := func(addr string, t cluster.Target) {
|
||||
if addr != "" {
|
||||
targets = append(targets, t)
|
||||
}
|
||||
}
|
||||
add(*pg, cluster.Target{Name: "PostgreSQL", Kind: cluster.TCP, Addr: *pg})
|
||||
add(*nats, cluster.Target{Name: "NATS", Kind: cluster.NATS, Addr: *nats})
|
||||
add(*etcd, cluster.Target{Name: "etcd", Kind: cluster.HTTP, Addr: "http://" + *etcd + "/health"})
|
||||
add(*patroni, cluster.Target{Name: "Patroni leader", Kind: cluster.HTTP, Addr: "http://" + *patroni + "/primary", HealthyStatus: 200})
|
||||
add(*s3, cluster.Target{Name: "S3 blobs", Kind: cluster.HTTP, Addr: "http://" + *s3 + "/health"})
|
||||
|
||||
if len(targets) == 0 {
|
||||
fmt.Fprintln(os.Stderr, "nothing to check: name an endpoint, such as --pg HOST:PORT (inbuxa cluster doctor -h)")
|
||||
return 2
|
||||
}
|
||||
|
||||
checks := cluster.Doctor(context.Background(), targets, *timeout)
|
||||
w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
|
||||
for _, c := range checks {
|
||||
mark := "ok"
|
||||
if !c.OK {
|
||||
mark = "unreachable"
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", c.Name, c.Addr, mark, c.Detail)
|
||||
}
|
||||
w.Flush()
|
||||
if !cluster.AllOK(checks) {
|
||||
fmt.Fprintln(os.Stderr, "\nsome endpoints didn't answer; a node can't join until they do")
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func relayInstall(ctx context.Context, args []string) int {
|
||||
fs := flag.NewFlagSet("relay install", flag.ContinueOnError)
|
||||
fs.Usage = func() { fmt.Print(usage) }
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
// SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
// Package cluster checks whether a node can reach the data layer a clustered
|
||||
// inbuxa server depends on: the PostgreSQL store, the NATS coordinator, the
|
||||
// S3 blob store, and the etcd and Patroni that stand behind them. It probes
|
||||
// only endpoints the caller names, so it fits any deployment; the probe each
|
||||
// one uses follows that service's own convention, not any one cluster's
|
||||
// layout.
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Kind is how an endpoint is probed.
|
||||
type Kind int
|
||||
|
||||
const (
|
||||
// TCP just opens a connection: the service is listening.
|
||||
TCP Kind = iota
|
||||
// NATS opens a connection and reads the server's INFO greeting.
|
||||
NATS
|
||||
// HTTP makes a GET and checks the status (HealthyStatus, or any 2xx when
|
||||
// that is zero).
|
||||
HTTP
|
||||
)
|
||||
|
||||
// Target is one endpoint to probe.
|
||||
type Target struct {
|
||||
Name string // what to call it in the report, e.g. "PostgreSQL"
|
||||
Kind Kind
|
||||
Addr string // host:port for TCP and NATS; a URL for HTTP
|
||||
// HealthyStatus is the HTTP status that means healthy; 0 accepts any 2xx.
|
||||
// For Patroni's /primary, 200 means this node is the leader and 503 that
|
||||
// it isn't, so the caller sets it to 200.
|
||||
HealthyStatus int
|
||||
}
|
||||
|
||||
// Check is the result of probing one target.
|
||||
type Check struct {
|
||||
Name string
|
||||
Addr string
|
||||
OK bool
|
||||
Detail string
|
||||
}
|
||||
|
||||
// Doctor probes every target, in order, and returns a Check for each. A nil
|
||||
// context uses the background context.
|
||||
func Doctor(ctx context.Context, targets []Target, timeout time.Duration) []Check {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
checks := make([]Check, 0, len(targets))
|
||||
for _, t := range targets {
|
||||
checks = append(checks, probe(ctx, t, timeout))
|
||||
}
|
||||
return checks
|
||||
}
|
||||
|
||||
// AllOK reports whether every check passed.
|
||||
func AllOK(checks []Check) bool {
|
||||
for _, c := range checks {
|
||||
if !c.OK {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func probe(ctx context.Context, t Target, timeout time.Duration) Check {
|
||||
c := Check{Name: t.Name, Addr: t.Addr}
|
||||
switch t.Kind {
|
||||
case NATS:
|
||||
c.OK, c.Detail = natsProbe(ctx, t.Addr, timeout)
|
||||
case HTTP:
|
||||
c.OK, c.Detail = httpProbe(ctx, t.Addr, t.HealthyStatus, timeout)
|
||||
default:
|
||||
c.OK, c.Detail = tcpProbe(ctx, t.Addr, timeout)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func tcpProbe(ctx context.Context, addr string, timeout time.Duration) (bool, string) {
|
||||
conn, err := dial(ctx, addr, timeout)
|
||||
if err != nil {
|
||||
return false, reason(err)
|
||||
}
|
||||
conn.Close()
|
||||
return true, "listening"
|
||||
}
|
||||
|
||||
// natsProbe confirms the port answers like NATS: the server sends an `INFO`
|
||||
// line as soon as a client connects.
|
||||
func natsProbe(ctx context.Context, addr string, timeout time.Duration) (bool, string) {
|
||||
conn, err := dial(ctx, addr, timeout)
|
||||
if err != nil {
|
||||
return false, reason(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
_ = conn.SetReadDeadline(time.Now().Add(timeout))
|
||||
line, err := bufio.NewReader(conn).ReadString('\n')
|
||||
if err != nil {
|
||||
return false, "connected, but no greeting: " + reason(err)
|
||||
}
|
||||
if strings.HasPrefix(strings.TrimSpace(line), "INFO") {
|
||||
return true, "NATS greeting"
|
||||
}
|
||||
return false, "connected, but the greeting wasn't NATS"
|
||||
}
|
||||
|
||||
func httpProbe(ctx context.Context, url string, want int, timeout time.Duration) (bool, string) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return false, reason(err)
|
||||
}
|
||||
resp, err := (&http.Client{Timeout: timeout}).Do(req)
|
||||
if err != nil {
|
||||
return false, reason(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
ok := resp.StatusCode == want || (want == 0 && resp.StatusCode >= 200 && resp.StatusCode < 300)
|
||||
return ok, fmt.Sprintf("HTTP %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
func dial(ctx context.Context, addr string, timeout time.Duration) (net.Conn, error) {
|
||||
return (&net.Dialer{Timeout: timeout}).DialContext(ctx, "tcp", addr)
|
||||
}
|
||||
|
||||
// reason trims a dial error to the part worth showing.
|
||||
func reason(err error) string {
|
||||
msg := err.Error()
|
||||
if i := strings.LastIndex(msg, ": "); i >= 0 {
|
||||
return msg[i+2:]
|
||||
}
|
||||
return msg
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
// SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const wait = 2 * time.Second
|
||||
|
||||
// listenTCP opens a throwaway TCP listener and returns its address. If greet
|
||||
// is non-empty, it's written to the first connection, like a NATS greeting.
|
||||
func listenTCP(t *testing.T, greet string) string {
|
||||
t.Helper()
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { ln.Close() })
|
||||
go func() {
|
||||
for {
|
||||
conn, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if greet != "" {
|
||||
conn.Write([]byte(greet))
|
||||
}
|
||||
conn.Close()
|
||||
}
|
||||
}()
|
||||
return ln.Addr().String()
|
||||
}
|
||||
|
||||
// closedAddr returns an address nothing listens on.
|
||||
func closedAddr(t *testing.T) string {
|
||||
t.Helper()
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
addr := ln.Addr().String()
|
||||
ln.Close()
|
||||
return addr
|
||||
}
|
||||
|
||||
func TestTCPProbe(t *testing.T) {
|
||||
open := listenTCP(t, "")
|
||||
if ok, _ := tcpProbe(context.Background(), open, wait); !ok {
|
||||
t.Error("a listening port should probe OK")
|
||||
}
|
||||
if ok, detail := tcpProbe(context.Background(), closedAddr(t), wait); ok {
|
||||
t.Errorf("a closed port should fail, got OK (%s)", detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNATSProbe(t *testing.T) {
|
||||
nats := listenTCP(t, "INFO {\"server_id\":\"x\"}\r\n")
|
||||
if ok, detail := natsProbe(context.Background(), nats, wait); !ok {
|
||||
t.Errorf("a NATS greeting should probe OK, got %q", detail)
|
||||
}
|
||||
// A port that's open but doesn't greet like NATS fails.
|
||||
notNats := listenTCP(t, "+OK\r\n")
|
||||
if ok, _ := natsProbe(context.Background(), notNats, wait); ok {
|
||||
t.Error("a non-NATS greeting should fail")
|
||||
}
|
||||
if ok, _ := natsProbe(context.Background(), closedAddr(t), wait); ok {
|
||||
t.Error("a closed port should fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPProbe(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/health":
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case "/primary":
|
||||
w.WriteHeader(http.StatusServiceUnavailable) // not the leader
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
// Any 2xx when the wanted status is 0.
|
||||
if ok, detail := httpProbe(context.Background(), srv.URL+"/health", 0, wait); !ok {
|
||||
t.Errorf("/health 200 should pass, got %q", detail)
|
||||
}
|
||||
// Patroni: 200 means leader; 503 (not leader) must fail when 200 is wanted.
|
||||
if ok, _ := httpProbe(context.Background(), srv.URL+"/primary", 200, wait); ok {
|
||||
t.Error("/primary 503 should fail when 200 is wanted")
|
||||
}
|
||||
if ok, _ := httpProbe(context.Background(), srv.URL+"/missing", 0, wait); ok {
|
||||
t.Error("404 should fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDoctorAndAllOK(t *testing.T) {
|
||||
open := listenTCP(t, "")
|
||||
checks := Doctor(nil, []Target{
|
||||
{Name: "PostgreSQL", Kind: TCP, Addr: open},
|
||||
{Name: "NATS", Kind: TCP, Addr: closedAddr(t)},
|
||||
}, wait)
|
||||
if len(checks) != 2 {
|
||||
t.Fatalf("want 2 checks, got %d", len(checks))
|
||||
}
|
||||
if !checks[0].OK || checks[1].OK {
|
||||
t.Errorf("unexpected results: %+v", checks)
|
||||
}
|
||||
if AllOK(checks) {
|
||||
t.Error("AllOK should be false when one check failed")
|
||||
}
|
||||
if !AllOK(checks[:1]) {
|
||||
t.Error("AllOK should be true when every check passed")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user