Files
jcoffey-dev 109f1bc780
ci / test (pull_request) Successful in 40s
cluster: a doctor that checks a node can reach the data layer
`inbuxa cluster doctor` probes the endpoints a clustered inbuxa server
depends on -- the PostgreSQL store, the NATS coordinator, etcd, Patroni
and the S3 blob store -- and reports which answer. It checks only the
endpoints named on the command line, each with that service's own probe
(a NATS greeting, etcd's /health, Patroni's /primary at 200 for the
leader), so it fits any deployment and touches nothing on the node.

It's the reachability preflight a node wants before it is pointed at a
cluster, and the groundwork for guided cluster membership.
2026-11-12 07:00:54 -07:00

144 lines
4.0 KiB
Go

// SPDX-FileCopyrightText: 2026 Coffey Labs LLC
// SPDX-License-Identifier: AGPL-3.0-or-later
// Package cluster checks whether a node can reach the data layer a clustered
// inbuxa server depends on: the PostgreSQL store, the NATS coordinator, the
// S3 blob store, and the etcd and Patroni that stand behind them. It probes
// only endpoints the caller names, so it fits any deployment; the probe each
// one uses follows that service's own convention, not any one cluster's
// layout.
package cluster
import (
"bufio"
"context"
"fmt"
"net"
"net/http"
"strings"
"time"
)
// Kind is how an endpoint is probed.
type Kind int
const (
// TCP just opens a connection: the service is listening.
TCP Kind = iota
// NATS opens a connection and reads the server's INFO greeting.
NATS
// HTTP makes a GET and checks the status (HealthyStatus, or any 2xx when
// that is zero).
HTTP
)
// Target is one endpoint to probe.
type Target struct {
Name string // what to call it in the report, e.g. "PostgreSQL"
Kind Kind
Addr string // host:port for TCP and NATS; a URL for HTTP
// HealthyStatus is the HTTP status that means healthy; 0 accepts any 2xx.
// For Patroni's /primary, 200 means this node is the leader and 503 that
// it isn't, so the caller sets it to 200.
HealthyStatus int
}
// Check is the result of probing one target.
type Check struct {
Name string
Addr string
OK bool
Detail string
}
// Doctor probes every target, in order, and returns a Check for each. A nil
// context uses the background context.
func Doctor(ctx context.Context, targets []Target, timeout time.Duration) []Check {
if ctx == nil {
ctx = context.Background()
}
checks := make([]Check, 0, len(targets))
for _, t := range targets {
checks = append(checks, probe(ctx, t, timeout))
}
return checks
}
// AllOK reports whether every check passed.
func AllOK(checks []Check) bool {
for _, c := range checks {
if !c.OK {
return false
}
}
return true
}
func probe(ctx context.Context, t Target, timeout time.Duration) Check {
c := Check{Name: t.Name, Addr: t.Addr}
switch t.Kind {
case NATS:
c.OK, c.Detail = natsProbe(ctx, t.Addr, timeout)
case HTTP:
c.OK, c.Detail = httpProbe(ctx, t.Addr, t.HealthyStatus, timeout)
default:
c.OK, c.Detail = tcpProbe(ctx, t.Addr, timeout)
}
return c
}
func tcpProbe(ctx context.Context, addr string, timeout time.Duration) (bool, string) {
conn, err := dial(ctx, addr, timeout)
if err != nil {
return false, reason(err)
}
conn.Close()
return true, "listening"
}
// natsProbe confirms the port answers like NATS: the server sends an `INFO`
// line as soon as a client connects.
func natsProbe(ctx context.Context, addr string, timeout time.Duration) (bool, string) {
conn, err := dial(ctx, addr, timeout)
if err != nil {
return false, reason(err)
}
defer conn.Close()
_ = conn.SetReadDeadline(time.Now().Add(timeout))
line, err := bufio.NewReader(conn).ReadString('\n')
if err != nil {
return false, "connected, but no greeting: " + reason(err)
}
if strings.HasPrefix(strings.TrimSpace(line), "INFO") {
return true, "NATS greeting"
}
return false, "connected, but the greeting wasn't NATS"
}
func httpProbe(ctx context.Context, url string, want int, timeout time.Duration) (bool, string) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return false, reason(err)
}
resp, err := (&http.Client{Timeout: timeout}).Do(req)
if err != nil {
return false, reason(err)
}
defer resp.Body.Close()
ok := resp.StatusCode == want || (want == 0 && resp.StatusCode >= 200 && resp.StatusCode < 300)
return ok, fmt.Sprintf("HTTP %d", resp.StatusCode)
}
func dial(ctx context.Context, addr string, timeout time.Duration) (net.Conn, error) {
return (&net.Dialer{Timeout: timeout}).DialContext(ctx, "tcp", addr)
}
// reason trims a dial error to the part worth showing.
func reason(err error) string {
msg := err.Error()
if i := strings.LastIndex(msg, ": "); i >= 0 {
return msg[i+2:]
}
return msg
}