Add an inbuxa doctor health check #60
No files matched your search
@@ -28,6 +28,7 @@ import (
|
||||
"git.coffeylabs.org/inbuxa/inbuxa-installer/internal/cluster"
|
||||
"git.coffeylabs.org/inbuxa/inbuxa-installer/internal/deps"
|
||||
"git.coffeylabs.org/inbuxa/inbuxa-installer/internal/discover"
|
||||
"git.coffeylabs.org/inbuxa/inbuxa-installer/internal/doctor"
|
||||
"git.coffeylabs.org/inbuxa/inbuxa-installer/internal/host"
|
||||
"git.coffeylabs.org/inbuxa/inbuxa-installer/internal/plan"
|
||||
"git.coffeylabs.org/inbuxa/inbuxa-installer/internal/relay"
|
||||
@@ -52,6 +53,9 @@ const usage = `inbuxa -- install the inbuxa suite on this machine
|
||||
inbuxa plan -f FILE what a topology file would change here
|
||||
inbuxa apply -f FILE make this machine match that file
|
||||
inbuxa status what is installed here, and whether it agrees
|
||||
inbuxa doctor a read-only health check: services, the mail
|
||||
server's certificate, the ports answering, and
|
||||
the DNS a sender needs (exit 1 on a failure)
|
||||
inbuxa upgrade [component…] move what is here to the releases this
|
||||
installer installs: --dry-run, --yes,
|
||||
--no-backup, --check (report only; exit 10 if
|
||||
@@ -208,6 +212,8 @@ func main() {
|
||||
os.Exit(exportCmd(os.Args[2:]))
|
||||
case "status":
|
||||
os.Exit(statusCmd())
|
||||
case "doctor":
|
||||
os.Exit(doctorCmd(os.Args[2:]))
|
||||
case "upgrade":
|
||||
os.Exit(upgradeCmd(os.Args[2:]))
|
||||
case "uninstall":
|
||||
@@ -1557,6 +1563,142 @@ func statusCmd() int {
|
||||
return 0
|
||||
}
|
||||
|
||||
// doctorCmd is a one-shot, read-only health check of the install on this
|
||||
// machine: the services that should run, the certificate the mail ports
|
||||
// present, those ports answering, and the DNS a sender needs. It changes
|
||||
// nothing, and exits 1 only when something has failed -- a warning, such as
|
||||
// DNS that has not spread yet, does not fail it.
|
||||
func doctorCmd(args []string) int {
|
||||
fs := flag.NewFlagSet("doctor", flag.ContinueOnError)
|
||||
fs.Usage = func() { fmt.Print(usage) }
|
||||
timeout := fs.Duration("timeout", 5*time.Second, "")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return 2
|
||||
}
|
||||
ctx := context.Background()
|
||||
st, err := state.Load()
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err.Error())
|
||||
return 1
|
||||
}
|
||||
f := host.Survey(ctx)
|
||||
found := discover.Run(ctx, f, st.Dir)
|
||||
if !st.Installed() && found.Dir == "" {
|
||||
fmt.Println("Nothing is installed on this machine; there is nothing to check.")
|
||||
return 0
|
||||
}
|
||||
if st.Removed() {
|
||||
fmt.Println("The programs were uninstalled here and the data kept; there is nothing running to check.")
|
||||
return 0
|
||||
}
|
||||
|
||||
var r doctor.Report
|
||||
for _, c := range doctor.ServiceChecks(st.Shapes, found.Services) {
|
||||
r.Add(c)
|
||||
}
|
||||
|
||||
_, serverHere := st.Shapes["server"]
|
||||
mailHost := apply.CertsMailHost("", st, found.MailHost)
|
||||
local := apply.LocalInstall(st)
|
||||
|
||||
if serverHere {
|
||||
// The certificate the mail ports present, unless this is --local.
|
||||
var problem string
|
||||
var perr error
|
||||
if !local {
|
||||
s, err := apply.ReadServed(ctx, mailHost, apply.CertRoots(st.Dir))
|
||||
if err != nil {
|
||||
perr = err
|
||||
} else {
|
||||
problem = s.Problem(mailHost, time.Now())
|
||||
}
|
||||
}
|
||||
r.Add(doctor.CertCheck(local, perr, problem))
|
||||
|
||||
// The ports the server should answer on, dialed on loopback.
|
||||
ports := []struct {
|
||||
n int
|
||||
for_ string
|
||||
}{
|
||||
{25, "SMTP, mail from other servers"},
|
||||
{465, "submissions, apps sending"},
|
||||
{993, "IMAPS, apps reading"},
|
||||
{443, "HTTPS, the front ends and JMAP"},
|
||||
}
|
||||
var probes []doctor.PortProbe
|
||||
for _, p := range ports {
|
||||
pr := doctor.PortProbe{Port: p.n, For: p.for_}
|
||||
conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", p.n), *timeout)
|
||||
if err == nil {
|
||||
pr.Open = true
|
||||
conn.Close()
|
||||
}
|
||||
probes = append(probes, pr)
|
||||
}
|
||||
for _, c := range doctor.PortChecks(probes) {
|
||||
r.Add(c)
|
||||
}
|
||||
}
|
||||
|
||||
// The DNS a sending server needs, for a public install that runs the
|
||||
// server here.
|
||||
df := doctor.DNSFacts{MailHost: mailHost}
|
||||
domain := st.Domain
|
||||
if domain == "" {
|
||||
domain = found.Domain
|
||||
}
|
||||
if serverHere && !local && domain != "" {
|
||||
df.Domain = domain
|
||||
if mxs, err := net.DefaultResolver.LookupMX(ctx, domain); err != nil {
|
||||
df.MXErr = resolverReason(err)
|
||||
} else {
|
||||
for _, mx := range mxs {
|
||||
df.MX = append(df.MX, strings.TrimSuffix(mx.Host, "."))
|
||||
}
|
||||
}
|
||||
if addrs, err := net.DefaultResolver.LookupHost(ctx, mailHost); err != nil {
|
||||
df.HostErr = resolverReason(err)
|
||||
} else {
|
||||
df.HostAddrs = addrs
|
||||
}
|
||||
}
|
||||
for _, c := range doctor.DNSChecks(df) {
|
||||
r.Add(c)
|
||||
}
|
||||
|
||||
printDoctorReport(r)
|
||||
return r.ExitCode()
|
||||
}
|
||||
|
||||
// resolverReason trims a resolver error to its reason, dropping the "lookup
|
||||
// name on server:" preamble a *net.DNSError carries.
|
||||
func resolverReason(err error) string {
|
||||
var d *net.DNSError
|
||||
if errors.As(err, &d) {
|
||||
return d.Err
|
||||
}
|
||||
return err.Error()
|
||||
}
|
||||
|
||||
// printDoctorReport prints each check, then one line on the whole.
|
||||
func printDoctorReport(r doctor.Report) {
|
||||
mark := map[doctor.Level]string{
|
||||
doctor.OK: "ok ", doctor.Warn: "warn", doctor.Fail: "FAIL", doctor.Skipped: "-- ",
|
||||
}
|
||||
for _, c := range r.Checks {
|
||||
fmt.Printf(" [%s] %-16s %s\n", mark[c.Level], c.Name, c.Detail)
|
||||
}
|
||||
switch r.Worst() {
|
||||
case doctor.Fail:
|
||||
fmt.Println("\nSomething above is wrong. `inbuxa status` shows what is installed, `inbuxa certs`")
|
||||
fmt.Println("looks after the certificate, and `inbuxa dns` prints the records the domain needs.")
|
||||
case doctor.Warn:
|
||||
fmt.Println("\nUp and running. The warnings above are worth a look -- DNS often just needs time.")
|
||||
default:
|
||||
fmt.Println("\nEverything checks out.")
|
||||
}
|
||||
}
|
||||
|
||||
func machineNames(t *topology.Topology) []string {
|
||||
var out []string
|
||||
for _, m := range t.Machines {
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
// SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
// Package doctor reports the health of an install on this machine in one
|
||||
// pass: the services that should be running, the certificate a mail client
|
||||
// is given, the mail ports answering, and the DNS a sender needs to reach
|
||||
// it. Every check only reads; nothing here changes the install.
|
||||
//
|
||||
// The checks are pure functions over facts the caller has already gathered,
|
||||
// so the command does the probing -- surveying services, reading the served
|
||||
// certificate, dialing the ports, resolving names -- and this decides what
|
||||
// each result means. That keeps the judgement testable without a live
|
||||
// install.
|
||||
package doctor
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Level is how a check came out.
|
||||
type Level int
|
||||
|
||||
const (
|
||||
// OK is healthy.
|
||||
OK Level = iota
|
||||
// Warn is worth a look but not broken: often something outside the
|
||||
// machine, like DNS that has not propagated yet.
|
||||
Warn
|
||||
// Fail is broken: the install is not serving mail as it should.
|
||||
Fail
|
||||
// Skipped is a check that does not apply here, such as certificates on a
|
||||
// --local install.
|
||||
Skipped
|
||||
)
|
||||
|
||||
func (l Level) String() string {
|
||||
switch l {
|
||||
case OK:
|
||||
return "ok"
|
||||
case Warn:
|
||||
return "warn"
|
||||
case Fail:
|
||||
return "fail"
|
||||
case Skipped:
|
||||
return "skip"
|
||||
}
|
||||
return "?"
|
||||
}
|
||||
|
||||
// Check is one thing looked at and what was found.
|
||||
type Check struct {
|
||||
Name string
|
||||
Level Level
|
||||
Detail string
|
||||
}
|
||||
|
||||
// Report is every check from a run.
|
||||
type Report struct {
|
||||
Checks []Check
|
||||
}
|
||||
|
||||
// Add appends a check.
|
||||
func (r *Report) Add(c Check) { r.Checks = append(r.Checks, c) }
|
||||
|
||||
// Worst is the most serious level in the report, OK when empty. Skipped does
|
||||
// not count as worse than OK.
|
||||
func (r Report) Worst() Level {
|
||||
worst := OK
|
||||
for _, c := range r.Checks {
|
||||
if c.Level == Skipped {
|
||||
continue
|
||||
}
|
||||
if c.Level > worst {
|
||||
worst = c.Level
|
||||
}
|
||||
}
|
||||
return worst
|
||||
}
|
||||
|
||||
// ExitCode is 1 when anything failed, else 0. A warning is not a failure: it
|
||||
// is a heads-up, and a script that gates on doctor should not stop for DNS
|
||||
// that is still spreading.
|
||||
func (r Report) ExitCode() int {
|
||||
if r.Worst() == Fail {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// ServiceChecks is one check per component the install means to run (its
|
||||
// shape is not "skip"/"none"/""), from the discovered service states
|
||||
// (component -> running|stopped|absent). A running service is OK; anything
|
||||
// else is a failure.
|
||||
func ServiceChecks(shapes, services map[string]string) []Check {
|
||||
var kinds []string
|
||||
for kind, shape := range shapes {
|
||||
switch strings.ToLower(shape) {
|
||||
case "", "skip", "no", "none":
|
||||
continue
|
||||
}
|
||||
kinds = append(kinds, kind)
|
||||
}
|
||||
sort.Strings(kinds)
|
||||
checks := make([]Check, 0, len(kinds))
|
||||
for _, kind := range kinds {
|
||||
name := "service: " + kind
|
||||
switch services[kind] {
|
||||
case "running":
|
||||
checks = append(checks, Check{name, OK, "running"})
|
||||
case "stopped":
|
||||
checks = append(checks, Check{name, Fail, "installed but stopped"})
|
||||
default: // "absent" or missing
|
||||
checks = append(checks, Check{name, Fail, "not running and not found in the deployment"})
|
||||
}
|
||||
}
|
||||
return checks
|
||||
}
|
||||
|
||||
// CertCheck judges the certificate a mail client is served. local skips it
|
||||
// (a --local install has no public certificate). A probe error -- the mail
|
||||
// ports not answering with TLS -- is a failure. Otherwise problem is what
|
||||
// the certificate checker found wrong, "" when nothing is: a certificate
|
||||
// that is merely close to expiry without renewing is a warning, and anything
|
||||
// else wrong (self-signed, expired, wrong name, untrusted) is a failure.
|
||||
func CertCheck(local bool, probeErr error, problem string) Check {
|
||||
const name = "certificate"
|
||||
switch {
|
||||
case local:
|
||||
return Check{name, Skipped, "a --local install serves no public certificate"}
|
||||
case probeErr != nil:
|
||||
return Check{name, Fail, probeErr.Error()}
|
||||
case problem == "":
|
||||
return Check{name, OK, "verifies and is current"}
|
||||
case strings.HasPrefix(problem, "it expires "):
|
||||
// The only soft case the checker reports: valid today, but past the
|
||||
// point it should have renewed.
|
||||
return Check{name, Warn, problem}
|
||||
default:
|
||||
return Check{name, Fail, problem}
|
||||
}
|
||||
}
|
||||
|
||||
// PortProbe is the result of dialing one port the server should answer on.
|
||||
type PortProbe struct {
|
||||
Port int
|
||||
For string // what the port is for, shown in the detail
|
||||
Open bool
|
||||
Err string // why it did not answer, when it did not
|
||||
}
|
||||
|
||||
// PortChecks is one check per probed port: open is OK, closed is a failure.
|
||||
func PortChecks(probes []PortProbe) []Check {
|
||||
checks := make([]Check, 0, len(probes))
|
||||
for _, p := range probes {
|
||||
name := fmt.Sprintf("port %d", p.Port)
|
||||
if p.Open {
|
||||
checks = append(checks, Check{name, OK, p.For})
|
||||
continue
|
||||
}
|
||||
detail := p.For + ": no answer"
|
||||
if p.Err != "" {
|
||||
detail = p.For + ": " + p.Err
|
||||
}
|
||||
checks = append(checks, Check{name, Fail, detail})
|
||||
}
|
||||
return checks
|
||||
}
|
||||
|
||||
// DNSFacts are the resolver answers the DNS checks judge.
|
||||
type DNSFacts struct {
|
||||
Domain string // the mail domain
|
||||
MailHost string // the server's name, where its MX should point
|
||||
MX []string // the domain's MX hosts, trailing dot trimmed
|
||||
MXErr string // why the MX lookup failed, when it did
|
||||
HostAddrs []string // the mail host's A/AAAA addresses
|
||||
HostErr string // why the host lookup failed, when it did
|
||||
}
|
||||
|
||||
// DNSChecks judges the DNS a sending server needs: the domain's MX, and that
|
||||
// the mail host it names resolves to an address. DNS that is missing or not
|
||||
// yet pointing at this host is a warning, not a failure: a fresh install is
|
||||
// healthy before its records are published or have spread. With no domain
|
||||
// (a --local install, or one whose domain was never recorded) the checks are
|
||||
// skipped.
|
||||
func DNSChecks(f DNSFacts) []Check {
|
||||
if f.Domain == "" {
|
||||
return []Check{{"dns", Skipped, "no mail domain to check"}}
|
||||
}
|
||||
var checks []Check
|
||||
|
||||
mx := Check{Name: "dns: MX"}
|
||||
switch {
|
||||
case f.MXErr != "":
|
||||
mx.Level, mx.Detail = Warn, f.Domain+" has no MX record yet ("+f.MXErr+")"
|
||||
case len(f.MX) == 0:
|
||||
mx.Level, mx.Detail = Warn, f.Domain+" has no MX record yet"
|
||||
case f.MailHost != "" && !containsFold(f.MX, f.MailHost):
|
||||
mx.Level, mx.Detail = Warn, f.Domain+" MX is "+strings.Join(f.MX, ", ")+", not "+f.MailHost
|
||||
default:
|
||||
mx.Level, mx.Detail = OK, f.Domain+" -> "+strings.Join(f.MX, ", ")
|
||||
}
|
||||
checks = append(checks, mx)
|
||||
|
||||
if f.MailHost != "" {
|
||||
host := Check{Name: "dns: mail host"}
|
||||
switch {
|
||||
case f.HostErr != "":
|
||||
host.Level, host.Detail = Warn, f.MailHost+" does not resolve yet ("+f.HostErr+")"
|
||||
case len(f.HostAddrs) == 0:
|
||||
host.Level, host.Detail = Warn, f.MailHost+" does not resolve yet"
|
||||
default:
|
||||
host.Level, host.Detail = OK, f.MailHost+" -> "+strings.Join(f.HostAddrs, ", ")
|
||||
}
|
||||
checks = append(checks, host)
|
||||
}
|
||||
return checks
|
||||
}
|
||||
|
||||
func containsFold(hay []string, needle string) bool {
|
||||
for _, h := range hay {
|
||||
if strings.EqualFold(strings.TrimSuffix(h, "."), strings.TrimSuffix(needle, ".")) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
// SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package doctor
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestServiceChecksSkipUnchosenAndFlagNotRunning(t *testing.T) {
|
||||
shapes := map[string]string{"server": "container", "console": "host", "webmail": "skip"}
|
||||
services := map[string]string{"server": "running", "console": "stopped", "webmail": "absent"}
|
||||
got := ServiceChecks(shapes, services)
|
||||
// webmail is "skip", so it is not checked; server and console are, in name order.
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("want 2 checks, got %d: %+v", len(got), got)
|
||||
}
|
||||
if got[0].Name != "service: console" || got[0].Level != Fail {
|
||||
t.Errorf("console should fail (stopped): %+v", got[0])
|
||||
}
|
||||
if got[1].Name != "service: server" || got[1].Level != OK {
|
||||
t.Errorf("server should be ok (running): %+v", got[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceChecksMissingServiceFails(t *testing.T) {
|
||||
got := ServiceChecks(map[string]string{"server": "container"}, map[string]string{})
|
||||
if len(got) != 1 || got[0].Level != Fail {
|
||||
t.Fatalf("a chosen but undiscovered service should fail: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCertCheck(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
local bool
|
||||
err error
|
||||
problem string
|
||||
want Level
|
||||
}{
|
||||
{"local skips", true, nil, "", Skipped},
|
||||
{"probe error fails", false, errors.New("no TLS"), "", Fail},
|
||||
{"clean is ok", false, nil, "", OK},
|
||||
{"expiring soon warns", false, nil, "it expires in 3 days and has not been renewed", Warn},
|
||||
{"expired fails", false, nil, "it expired on 2026-01-01", Fail},
|
||||
{"self-signed fails", false, nil, "it is the server's own self-signed certificate, which mail clients refuse", Fail},
|
||||
{"wrong name fails", false, nil, "it does not name mail.example.com", Fail},
|
||||
{"untrusted fails", false, nil, "it does not verify: x509: unknown authority", Fail},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
got := CertCheck(c.local, c.err, c.problem)
|
||||
if got.Level != c.want {
|
||||
t.Errorf("CertCheck(%v,%v,%q) level = %v, want %v", c.local, c.err, c.problem, got.Level, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPortChecks(t *testing.T) {
|
||||
got := PortChecks([]PortProbe{
|
||||
{Port: 25, For: "SMTP", Open: true},
|
||||
{Port: 993, For: "IMAPS", Open: false, Err: "connection refused"},
|
||||
})
|
||||
if got[0].Level != OK {
|
||||
t.Errorf("open port should be ok: %+v", got[0])
|
||||
}
|
||||
if got[1].Level != Fail || got[1].Name != "port 993" {
|
||||
t.Errorf("closed port should fail: %+v", got[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDNSChecks(t *testing.T) {
|
||||
t.Run("no domain is skipped", func(t *testing.T) {
|
||||
got := DNSChecks(DNSFacts{})
|
||||
if len(got) != 1 || got[0].Level != Skipped {
|
||||
t.Fatalf("want one skipped check: %+v", got)
|
||||
}
|
||||
})
|
||||
t.Run("matching MX and resolving host are ok", func(t *testing.T) {
|
||||
got := DNSChecks(DNSFacts{
|
||||
Domain: "example.com", MailHost: "mail.example.com.",
|
||||
MX: []string{"mail.example.com."}, HostAddrs: []string{"203.0.113.4"},
|
||||
})
|
||||
for _, c := range got {
|
||||
if c.Level != OK {
|
||||
t.Errorf("%s should be ok: %+v", c.Name, c)
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run("missing MX and unresolved host warn, not fail", func(t *testing.T) {
|
||||
got := DNSChecks(DNSFacts{
|
||||
Domain: "example.com", MailHost: "mail.example.com",
|
||||
MXErr: "no such host", HostErr: "no such host",
|
||||
})
|
||||
for _, c := range got {
|
||||
if c.Level != Warn {
|
||||
t.Errorf("%s should warn: %+v", c.Name, c)
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run("MX pointing elsewhere warns", func(t *testing.T) {
|
||||
got := DNSChecks(DNSFacts{
|
||||
Domain: "example.com", MailHost: "mail.example.com",
|
||||
MX: []string{"aspmx.l.google.com."}, HostAddrs: []string{"203.0.113.4"},
|
||||
})
|
||||
if got[0].Level != Warn {
|
||||
t.Errorf("MX pointing elsewhere should warn: %+v", got[0])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestReportWorstAndExit(t *testing.T) {
|
||||
r := Report{}
|
||||
r.Add(Check{"a", OK, ""})
|
||||
r.Add(Check{"b", Skipped, ""})
|
||||
if r.Worst() != OK || r.ExitCode() != 0 {
|
||||
t.Fatalf("ok+skipped should be ok/exit 0: worst=%v exit=%d", r.Worst(), r.ExitCode())
|
||||
}
|
||||
r.Add(Check{"c", Warn, ""})
|
||||
if r.Worst() != Warn || r.ExitCode() != 0 {
|
||||
t.Fatalf("a warning is not a failure: worst=%v exit=%d", r.Worst(), r.ExitCode())
|
||||
}
|
||||
r.Add(Check{"d", Fail, ""})
|
||||
if r.Worst() != Fail || r.ExitCode() != 1 {
|
||||
t.Fatalf("a failure exits 1: worst=%v exit=%d", r.Worst(), r.ExitCode())
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user