Services: one copy down of several is degraded, not down #53
No files matched your search
+6
-1
@@ -784,7 +784,12 @@ makes up its state: `checks` (ids), `containers` and `units`
|
||||
`cloudflare: true`. Each service claims the containers and units its
|
||||
patterns match, which take their place under its own `id` and `name`;
|
||||
one nothing matches is still drawn, as declared. A service's state is the
|
||||
worst of its parts, and its history is kept for the Reports page.
|
||||
worst of its parts, and its history is kept for the Reports page. One
|
||||
exception: a unit or container the service has on several hosts is
|
||||
redundant, so a copy that's down while others still run makes the service
|
||||
degraded ("1 of 6 redis-server.service down"), not down; when every copy
|
||||
is down, or there's only one, the service is down. Each copy's own alert
|
||||
is unchanged.
|
||||
|
||||
Up to 0.1.0-rc.3 this file also drew two hand-drawn maps, with `mesh_ip`,
|
||||
`mesh_check`, `network.mail`, a remote host's `via`, `link` and
|
||||
|
||||
@@ -253,6 +253,7 @@ func (srv *Server) serviceStates(now time.Time) (_ []ServiceView, ok bool) {
|
||||
// Parts still waiting for their first result don't grey out a
|
||||
// service whose other parts have answered.
|
||||
v.Status, v.Word = "good", "OK"
|
||||
spare := redundantDown(v.Signals)
|
||||
bad, answered := 0, 0
|
||||
off := 0
|
||||
for _, sg := range v.Signals {
|
||||
@@ -264,7 +265,11 @@ func (srv *Server) serviceStates(now time.Time) (_ []ServiceView, ok bool) {
|
||||
off++
|
||||
continue
|
||||
}
|
||||
v.Status = worst(v.Status, sg.Status)
|
||||
st := sg.Status
|
||||
if st == "critical" && spare[sg.Kind+"/"+sg.Name] != "" {
|
||||
st = "warning" // other copies carry on (#52)
|
||||
}
|
||||
v.Status = worst(v.Status, st)
|
||||
if sg.Status == "warning" || sg.Status == "critical" {
|
||||
bad++
|
||||
}
|
||||
@@ -280,11 +285,47 @@ func (srv *Server) serviceStates(now time.Time) (_ []ServiceView, ok bool) {
|
||||
if bad > 0 {
|
||||
v.Word = plural(bad, "problem", "problems")
|
||||
}
|
||||
if v.Status == "warning" && bad == 1 {
|
||||
for _, w := range spare {
|
||||
v.Word = w
|
||||
}
|
||||
}
|
||||
services = append(services, v)
|
||||
}
|
||||
return services, true
|
||||
}
|
||||
|
||||
// redundantDown is, for each unit or container a service has several
|
||||
// copies of (one name on several hosts), the word for the copies that are
|
||||
// down while others still run: "1 of 6 redis-server down". Those make the
|
||||
// service degraded, not down; when every copy is down, or there's only
|
||||
// one, it's down (#52).
|
||||
func redundantDown(sigs []Signal) map[string]string {
|
||||
type count struct{ all, down int }
|
||||
n := map[string]*count{}
|
||||
for _, sg := range sigs {
|
||||
if sg.Kind != "unit" && sg.Kind != "container" || sg.Status == "waiting" || sg.Status == "off" {
|
||||
continue
|
||||
}
|
||||
k := sg.Kind + "/" + sg.Name
|
||||
if n[k] == nil {
|
||||
n[k] = &count{}
|
||||
}
|
||||
n[k].all++
|
||||
if sg.Status == "critical" {
|
||||
n[k].down++
|
||||
}
|
||||
}
|
||||
out := map[string]string{}
|
||||
for k, c := range n {
|
||||
if c.down > 0 && c.down < c.all {
|
||||
_, name, _ := strings.Cut(k, "/")
|
||||
out[k] = fmt.Sprintf("%d of %d %s down", c.down, c.all, name)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func nonNilSlice[T any](s []T) []T {
|
||||
if s == nil {
|
||||
return []T{}
|
||||
|
||||
@@ -191,3 +191,36 @@ func TestServiceMapJudgesByUnits(t *testing.T) {
|
||||
t.Errorf("patroni failed: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// One copy of a unit down while another runs is the service degraded,
|
||||
// not down; every copy down is down (#52).
|
||||
func TestServiceRedundantCopies(t *testing.T) {
|
||||
srv := topoServer(t)
|
||||
now := time.Now()
|
||||
mail := func() ServiceView {
|
||||
t.Helper()
|
||||
svcs, ok := srv.serviceStates(now)
|
||||
if !ok {
|
||||
t.Fatal("no services")
|
||||
}
|
||||
for _, s := range svcs {
|
||||
if s.ID == "mail" {
|
||||
return s
|
||||
}
|
||||
}
|
||||
t.Fatal("no mail service")
|
||||
return ServiceView{}
|
||||
}
|
||||
unit := func(active string) report.Report {
|
||||
return report.Report{V: report.Version, Units: []report.Unit{{Name: "inbuxa.service", Load: "loaded", Active: active, Sub: active, Since: now}}}
|
||||
}
|
||||
srv.store.Put("node1", unit("failed"), now)
|
||||
srv.store.Put("node2", unit("active"), now)
|
||||
if m := mail(); m.Status != "warning" || m.Word != "1 of 2 inbuxa.service down" {
|
||||
t.Errorf("one of two down: %s %q", m.Status, m.Word)
|
||||
}
|
||||
srv.store.Put("node2", unit("failed"), now)
|
||||
if m := mail(); m.Status != "critical" {
|
||||
t.Errorf("both down: %s %q", m.Status, m.Word)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user