Services: one copy down of several is degraded, not down #53

Merged
jcoffey-dev merged 1 commits from fix/service-redundant-parts into main 2026-11-16 06:19:10 +00:00
3 changed files with 81 additions and 2 deletions

No files matched your search

+6 -1
View File
@@ -784,7 +784,12 @@ makes up its state: `checks` (ids), `containers` and `units`
`cloudflare: true`. Each service claims the containers and units its
patterns match, which take their place under its own `id` and `name`;
one nothing matches is still drawn, as declared. A service's state is the
worst of its parts, and its history is kept for the Reports page.
worst of its parts, and its history is kept for the Reports page. One
exception: a unit or container the service has on several hosts is
redundant, so a copy that's down while others still run makes the service
degraded ("1 of 6 redis-server.service down"), not down; when every copy
is down, or there's only one, the service is down. Each copy's own alert
is unchanged.
Up to 0.1.0-rc.3 this file also drew two hand-drawn maps, with `mesh_ip`,
`mesh_check`, `network.mail`, a remote host's `via`, `link` and
+42 -1
View File
@@ -253,6 +253,7 @@ func (srv *Server) serviceStates(now time.Time) (_ []ServiceView, ok bool) {
// Parts still waiting for their first result don't grey out a
// service whose other parts have answered.
v.Status, v.Word = "good", "OK"
spare := redundantDown(v.Signals)
bad, answered := 0, 0
off := 0
for _, sg := range v.Signals {
@@ -264,7 +265,11 @@ func (srv *Server) serviceStates(now time.Time) (_ []ServiceView, ok bool) {
off++
continue
}
v.Status = worst(v.Status, sg.Status)
st := sg.Status
if st == "critical" && spare[sg.Kind+"/"+sg.Name] != "" {
st = "warning" // other copies carry on (#52)
}
v.Status = worst(v.Status, st)
if sg.Status == "warning" || sg.Status == "critical" {
bad++
}
@@ -280,11 +285,47 @@ func (srv *Server) serviceStates(now time.Time) (_ []ServiceView, ok bool) {
if bad > 0 {
v.Word = plural(bad, "problem", "problems")
}
if v.Status == "warning" && bad == 1 {
for _, w := range spare {
v.Word = w
}
}
services = append(services, v)
}
return services, true
}
// redundantDown is, for each unit or container a service has several
// copies of (one name on several hosts), the word for the copies that are
// down while others still run: "1 of 6 redis-server down". Those make the
// service degraded, not down; when every copy is down, or there's only
// one, it's down (#52).
func redundantDown(sigs []Signal) map[string]string {
type count struct{ all, down int }
n := map[string]*count{}
for _, sg := range sigs {
if sg.Kind != "unit" && sg.Kind != "container" || sg.Status == "waiting" || sg.Status == "off" {
continue
}
k := sg.Kind + "/" + sg.Name
if n[k] == nil {
n[k] = &count{}
}
n[k].all++
if sg.Status == "critical" {
n[k].down++
}
}
out := map[string]string{}
for k, c := range n {
if c.down > 0 && c.down < c.all {
_, name, _ := strings.Cut(k, "/")
out[k] = fmt.Sprintf("%d of %d %s down", c.down, c.all, name)
}
}
return out
}
func nonNilSlice[T any](s []T) []T {
if s == nil {
return []T{}
+33
View File
@@ -191,3 +191,36 @@ func TestServiceMapJudgesByUnits(t *testing.T) {
t.Errorf("patroni failed: %s", got)
}
}
// One copy of a unit down while another runs is the service degraded,
// not down; every copy down is down (#52).
func TestServiceRedundantCopies(t *testing.T) {
srv := topoServer(t)
now := time.Now()
mail := func() ServiceView {
t.Helper()
svcs, ok := srv.serviceStates(now)
if !ok {
t.Fatal("no services")
}
for _, s := range svcs {
if s.ID == "mail" {
return s
}
}
t.Fatal("no mail service")
return ServiceView{}
}
unit := func(active string) report.Report {
return report.Report{V: report.Version, Units: []report.Unit{{Name: "inbuxa.service", Load: "loaded", Active: active, Sub: active, Since: now}}}
}
srv.store.Put("node1", unit("failed"), now)
srv.store.Put("node2", unit("active"), now)
if m := mail(); m.Status != "warning" || m.Word != "1 of 2 inbuxa.service down" {
t.Errorf("one of two down: %s %q", m.Status, m.Word)
}
srv.store.Put("node2", unit("failed"), now)
if m := mail(); m.Status != "critical" {
t.Errorf("both down: %s %q", m.Status, m.Word)
}
}