diff --git a/docs/formats.md b/docs/formats.md index 2e3c939..5e09d7c 100644 --- a/docs/formats.md +++ b/docs/formats.md @@ -784,7 +784,12 @@ makes up its state: `checks` (ids), `containers` and `units` `cloudflare: true`. Each service claims the containers and units its patterns match, which take their place under its own `id` and `name`; one nothing matches is still drawn, as declared. A service's state is the -worst of its parts, and its history is kept for the Reports page. +worst of its parts, and its history is kept for the Reports page. One +exception: a unit or container the service has on several hosts is +redundant, so a copy that's down while others still run makes the service +degraded ("1 of 6 redis-server.service down"), not down; when every copy +is down, or there's only one, the service is down. Each copy's own alert +is unchanged. Up to 0.1.0-rc.3 this file also drew two hand-drawn maps, with `mesh_ip`, `mesh_check`, `network.mail`, a remote host's `via`, `link` and diff --git a/internal/server/topology.go b/internal/server/topology.go index dc4cd36..06df2f4 100644 --- a/internal/server/topology.go +++ b/internal/server/topology.go @@ -253,6 +253,7 @@ func (srv *Server) serviceStates(now time.Time) (_ []ServiceView, ok bool) { // Parts still waiting for their first result don't grey out a // service whose other parts have answered. v.Status, v.Word = "good", "OK" + spare := redundantDown(v.Signals) bad, answered := 0, 0 off := 0 for _, sg := range v.Signals { @@ -264,7 +265,11 @@ func (srv *Server) serviceStates(now time.Time) (_ []ServiceView, ok bool) { off++ continue } - v.Status = worst(v.Status, sg.Status) + st := sg.Status + if st == "critical" && spare[sg.Kind+"/"+sg.Name] != "" { + st = "warning" // other copies carry on (#52) + } + v.Status = worst(v.Status, st) if sg.Status == "warning" || sg.Status == "critical" { bad++ } @@ -280,11 +285,47 @@ func (srv *Server) serviceStates(now time.Time) (_ []ServiceView, ok bool) { if bad > 0 { v.Word = plural(bad, "problem", "problems") } + if v.Status == "warning" && bad == 1 { + for _, w := range spare { + v.Word = w + } + } services = append(services, v) } return services, true } +// redundantDown is, for each unit or container a service has several +// copies of (one name on several hosts), the word for the copies that are +// down while others still run: "1 of 6 redis-server down". Those make the +// service degraded, not down; when every copy is down, or there's only +// one, it's down (#52). +func redundantDown(sigs []Signal) map[string]string { + type count struct{ all, down int } + n := map[string]*count{} + for _, sg := range sigs { + if sg.Kind != "unit" && sg.Kind != "container" || sg.Status == "waiting" || sg.Status == "off" { + continue + } + k := sg.Kind + "/" + sg.Name + if n[k] == nil { + n[k] = &count{} + } + n[k].all++ + if sg.Status == "critical" { + n[k].down++ + } + } + out := map[string]string{} + for k, c := range n { + if c.down > 0 && c.down < c.all { + _, name, _ := strings.Cut(k, "/") + out[k] = fmt.Sprintf("%d of %d %s down", c.down, c.all, name) + } + } + return out +} + func nonNilSlice[T any](s []T) []T { if s == nil { return []T{} diff --git a/internal/server/topology_test.go b/internal/server/topology_test.go index 695cf88..a973019 100644 --- a/internal/server/topology_test.go +++ b/internal/server/topology_test.go @@ -191,3 +191,36 @@ func TestServiceMapJudgesByUnits(t *testing.T) { t.Errorf("patroni failed: %s", got) } } + +// One copy of a unit down while another runs is the service degraded, +// not down; every copy down is down (#52). +func TestServiceRedundantCopies(t *testing.T) { + srv := topoServer(t) + now := time.Now() + mail := func() ServiceView { + t.Helper() + svcs, ok := srv.serviceStates(now) + if !ok { + t.Fatal("no services") + } + for _, s := range svcs { + if s.ID == "mail" { + return s + } + } + t.Fatal("no mail service") + return ServiceView{} + } + unit := func(active string) report.Report { + return report.Report{V: report.Version, Units: []report.Unit{{Name: "inbuxa.service", Load: "loaded", Active: active, Sub: active, Since: now}}} + } + srv.store.Put("node1", unit("failed"), now) + srv.store.Put("node2", unit("active"), now) + if m := mail(); m.Status != "warning" || m.Word != "1 of 2 inbuxa.service down" { + t.Errorf("one of two down: %s %q", m.Status, m.Word) + } + srv.store.Put("node2", unit("failed"), now) + if m := mail(); m.Status != "critical" { + t.Errorf("both down: %s %q", m.Status, m.Word) + } +}