Alerts: alike problems from one place are one incident #50

Merged
jcoffey-dev merged 1 commits from feature/alert-batches into main 2026-11-16 06:03:29 +00:00
8 changed files with 228 additions and 9 deletions

No files matched your search

+22
View File
@@ -821,6 +821,28 @@ links to where it is on the maps. What it holds back and why is kept
with the incident, so every copy shows it. The dashboard groups the same
way, and the service map names the likely cause on a failing service.
### Batches
After grouping by cause, problems that are alike and from one place are
one incident when there are three or more at once: "4 certificates from
pki_int in vault-prod end soon", "3 VMs in vcenter-phx: inaccessible".
Its detail names them, worst first (eight at most, then "and 5 more"),
and it takes the worst one's severity. What's alike:
| batch | what's in it |
|---|---|
| certificates ending, by issuer | the Certificates page's own alerts (hosts' ports and files), when ending is all that's wrong |
| certificates ending, by Vault mount or cluster namespace | Vault PKI leaves and issuers; cert-manager's certificates |
| Akamai certificates ending | CPS's deployed certificates |
| VMs, by cluster and state | a vCenter's, Nutanix's, Proxmox's or KubeVirt's VMs failing the same way |
| workloads, by cluster and how their pods fail | Deployments, StatefulSets and the like whose pods all fail the same way (CrashLoopBackOff, say) |
The safety rules are grouping's: one that has already paged on its own
stays on its own, and a batch that has paged stays one incident while
any of it is left, so it neither resolves early nor opens again as
something else. Certificates that have ended, or are weak or served
wrong as well, are never batched: each is its own job.
## expected.yaml: what should run where
Per host, `running` and `stopped` name containers; `units` names systemd
+2 -1
View File
@@ -82,7 +82,8 @@ func (srv *Server) akamaiConditions(now time.Time) []Condition {
if st, word := certStatus(c, now, srv.certCritical()); st != "good" {
out = append(out, Condition{Key: key + "/" + c.Fingerprint, Severity: st, Ends: c.NotAfter,
Title: "Akamai certificate for " + e.CN + ": " + lowerFirst(word),
Detail: akamaiEnrollment(e) + ", " + c.Key + ". " + akamaiRenewal(e), Href: href})
Detail: akamaiEnrollment(e) + ", " + c.Key + ". " + akamaiRenewal(e), Href: href,
Batch: &Batch{Key: "akamai/cps-ends", Title: "{n} Akamai certificates end soon", Item: e.CN + " (" + certLeft(c.NotAfter.Sub(now)) + ")", Href: href}})
}
}
if what := akamaiWaiting[e.Waiting]; what != "" {
+2
View File
@@ -220,6 +220,8 @@ type Condition struct {
// Ends is when the certificate a condition is about ends, for one that
// is: certSeverity keeps it a warning until it's close (#34).
Ends time.Time `json:"-"`
// Batch folds it with others alike from the same place (batches.go).
Batch *Batch `json:"-"`
}
// view judges every host now: what the UI shows, and the conditions the
+110
View File
@@ -0,0 +1,110 @@
package server
import (
"cmp"
"slices"
"sort"
"strconv"
"strings"
"github.com/cairnobs/cairnobs/internal/notify"
)
// Batches (#33): several problems of one kind from one place at once (the
// certificates of one Vault mount ending in the same week, a vCenter's
// VMs all failing the same way) are one incident that lists them, not one
// page each. Root-cause grouping (grouping.go) folds what depends on a
// failure; this folds what's alike.
//
// The safety rules are grouping's: a member that has already paged on
// its own stays on its own, so it neither vanishes nor reads as resolved;
// the batch takes its worst member's severity, so one that gets worse
// escalates it; and a batch that has paged keeps its incident while any
// member is left, rather than resolving and opening again as one alert.
// Batch says which batch a condition belongs to.
type Batch struct {
Key string // what's alike and where: "certends/vault-prod/pki_int"
Title string // the batch's title, {n} for how many: "{n} certificates from pki_int in vault-prod end soon"
Item string // this one in the batch's list: "svc-102.prod.internal (ends in 5 days)"
Href string // where the batch's link goes (the first member's link without it)
}
// batchMin is how many alike make a batch.
const batchMin = 3
// batchListed is how many members a batch's detail names.
const batchListed = 8
// foldBatches replaces each batch's members with one condition, keyed
// batch/<key>. firing says whether an incident has paged.
func foldBatches(conds []Condition, firing func(key string) bool) []Condition {
members := map[string][]int{}
for i, c := range conds {
if c.Batch != nil && !firing(c.Key) {
members[c.Batch.Key] = append(members[c.Batch.Key], i)
}
}
if len(members) == 0 {
return conds
}
keys := make([]string, 0, len(members))
for k := range members {
keys = append(keys, k)
}
slices.Sort(keys)
folded := make([]bool, len(conds))
batch := map[int]Condition{} // by its first member's place in conds, so it takes that place
for _, k := range keys {
idx := members[k]
if len(idx) < batchMin && !firing("batch/"+k) {
continue
}
at := slices.Min(idx) // where the batch goes in the list
// Worst first, then by key: the same order from one evaluation to
// the next, so the detail doesn't shuffle.
sort.SliceStable(idx, func(a, b int) bool {
ca, cb := conds[idx[a]], conds[idx[b]]
if ra, rb := notify.Rank(ca.Severity), notify.Rank(cb.Severity); ra != rb {
return ra > rb
}
return ca.Key < cb.Key
})
first := conds[idx[0]]
b := Condition{Key: "batch/" + k, Severity: first.Severity, Href: cmp.Or(first.Batch.Href, first.Href), Map: first.Map, Host: first.Host, Ends: first.Ends,
Title: strings.ReplaceAll(first.Batch.Title, "{n}", strconv.Itoa(len(idx)))}
var items []string
for _, i := range idx {
c := conds[i]
folded[i] = true
if c.Host != b.Host {
b.Host = "" // a maintenance window holds it only when it's all one host's
}
if !c.Ends.IsZero() && (b.Ends.IsZero() || c.Ends.Before(b.Ends)) {
b.Ends = c.Ends
}
if len(items) < batchListed {
items = append(items, c.Batch.Item)
}
}
b.Detail = strings.Join(items, ", ")
if more := len(idx) - len(items); more > 0 {
b.Detail += " and " + strconv.Itoa(more) + " more"
}
b.Detail += "."
if len(idx) == 1 { // what's left of a batch that paged: still its incident
b.Title, b.Detail = first.Title, first.Detail
}
batch[at] = b
}
out := make([]Condition, 0, len(conds))
for i, c := range conds {
if b, ok := batch[i]; ok {
out = append(out, b)
}
if !folded[i] {
out = append(out, c)
}
}
return out
}
+58
View File
@@ -0,0 +1,58 @@
package server
import (
"strings"
"testing"
)
func batched(key, sev, item string) Condition {
return Condition{Key: key, Severity: sev, Title: key + " ends soon", Href: "/" + key,
Batch: &Batch{Key: "certends/vault-prod/pki_int", Title: "{n} certificates from pki_int end soon", Item: item, Href: "/workloads"}}
}
// Three or more alike are one condition, named once each; fewer stay as
// they are; one that has paged on its own stays on its own (#33).
func TestFoldBatches(t *testing.T) {
none := func(string) bool { return false }
other := Condition{Key: "host/node1", Severity: "critical", Title: "node1 is down"}
conds := []Condition{
batched("c", "warning", "c (6 days)"), other,
batched("a", "warning", "a (5 days)"),
batched("b", "critical", "b (30 hours)"),
}
out := foldBatches(conds, none)
if len(out) != 2 || out[0].Key != "batch/certends/vault-prod/pki_int" || out[1].Key != "host/node1" {
t.Fatalf("%+v", out)
}
b := out[0]
if b.Severity != "critical" || b.Title != "3 certificates from pki_int end soon" || b.Detail != "b (30 hours), a (5 days), c (6 days)." || b.Href != "/workloads" {
t.Errorf("batch: %+v", b)
}
if out := foldBatches(conds[:3], none); len(out) != 3 {
t.Errorf("two alike aren't a batch: %+v", out)
}
// b paged on its own before the others came: it stays, and the rest
// are too few.
paged := func(k string) bool { return k == "b" }
if out := foldBatches(conds, paged); len(out) != 4 {
t.Errorf("a paged member stays on its own: %+v", out)
}
// A batch that paged keeps its incident while one member is left.
batchPaged := func(k string) bool { return k == "batch/certends/vault-prod/pki_int" }
out = foldBatches([]Condition{batched("a", "warning", "a (5 days)")}, batchPaged)
if len(out) != 1 || out[0].Key != "batch/certends/vault-prod/pki_int" || out[0].Title != "a ends soon" {
t.Errorf("what's left of a paged batch: %+v", out)
}
// More than batchListed members: the rest are counted.
var many []Condition
for i := range 11 {
many = append(many, batched(string(rune('a'+i)), "warning", string(rune('a'+i))))
}
if out := foldBatches(many, none); len(out) != 1 || !strings.HasSuffix(out[0].Detail, "h and 3 more.") {
t.Errorf("many: %+v", out)
}
}
+5
View File
@@ -3,6 +3,7 @@ package server
import (
"net"
"net/http"
"net/url"
"slices"
"strconv"
"strings"
@@ -571,6 +572,10 @@ func (srv *Server) certConditions(now time.Time) []Condition {
cond.Host = h // a maintenance window on the host holds it
}
}
if own && c.Weak == "" && len(c.NotReloaded) == 0 && !wrongOnHosts(c) && c.Issuer != "" && c.NotAfter.After(now) {
cond.Batch = &Batch{Key: "certends/issuer/" + c.Issuer, Title: "{n} certificates from " + c.Issuer + " end soon",
Item: c.Subject + " (" + certLeft(c.NotAfter.Sub(now)) + ")", Href: "/certificates?q=" + url.QueryEscape(c.Issuer)}
}
conds = append(conds, cond)
}
return conds
+8 -2
View File
@@ -323,12 +323,18 @@ func nodeOf(t *Topology, c Condition) string {
return ""
}
// groupConditions folds conditions caused by a deeper failure into the
// groupConditions folds conditions by cause, then folds what's alike into
// batches (batches.go).
func (srv *Server) groupConditions(conds []Condition, firing func(key string) bool) []Condition {
return foldBatches(srv.groupByCause(conds, firing), firing)
}
// groupByCause folds conditions caused by a deeper failure into the
// condition for that failure. firing says whether a key already has an
// incident that people were told about: such a condition stays on its
// own, so it neither vanishes nor reads as "resolved" because its cause
// was spotted later.
func (srv *Server) groupConditions(conds []Condition, firing func(key string) bool) []Condition {
func (srv *Server) groupByCause(conds []Condition, firing func(key string) bool) []Condition {
if len(conds) == 0 {
return conds
}
+21 -6
View File
@@ -856,8 +856,13 @@ func judgeRecords(s wsnap, now time.Time, firing func(key string) bool, ad adopt
detail = append(detail, "Served on "+on+".")
}
ends, _ := time.Parse(time.RFC3339, fmt.Sprint(r.Facts["ends"]))
add(Condition{Key: "cert/" + src + "/" + r.ID, Severity: sev, Detail: strings.Join(detail, " "), Ends: ends,
Title: fmt.Sprintf("%s %s%s in %s: %s", what, r.Name, from, name, lowerFirst(r.Word)), Href: workloadHref(src, r.Namespace, r.Name)}, ad.alert(r, r.Alert), r.Kind)
c := Condition{Key: "cert/" + src + "/" + r.ID, Severity: sev, Detail: strings.Join(detail, " "), Ends: ends,
Title: fmt.Sprintf("%s %s%s in %s: %s", what, r.Name, from, name, lowerFirst(r.Word)), Href: workloadHref(src, r.Namespace, r.Name)}
if strings.HasPrefix(r.Word, "Ends in") && !strings.Contains(r.Word, ";") && !ends.IsZero() { // ending, and nothing else wrong: alike
c.Batch = &Batch{Key: "certends/" + src + "/" + r.Namespace, Title: "{n} certificates" + from + " in " + name + " end soon", Href: workloadHref(src, r.Namespace, ""),
Item: r.Name + " (" + certLeft(ends.Sub(now)) + ")"}
}
add(c, ad.alert(r, r.Alert), r.Kind)
}
case report.KindVM:
sev, detail := "", ""
@@ -876,7 +881,8 @@ func judgeRecords(s wsnap, now time.Time, firing func(key string) bool, ad adopt
}
if sev != "" {
add(Condition{Key: "vm/" + src + "/" + r.Namespace + "/" + r.Name, Severity: sev, Detail: detail,
Title: fmt.Sprintf("VM %s (%s): %s", where(r.Name, r.Namespace), name, lowerFirst(r.Word)), Href: workloadHref(src, r.Namespace, r.Name)}, ad.alert(r, r.Alert), r.Kind)
Title: fmt.Sprintf("VM %s (%s): %s", where(r.Name, r.Namespace), name, lowerFirst(r.Word)), Href: workloadHref(src, r.Namespace, r.Name),
Batch: &Batch{Key: "vm/" + src + "/" + r.Word, Title: "{n} VMs in " + name + ": " + lowerFirst(r.Word), Item: where(r.Name, r.Namespace), Href: workloadHref(src, "", "")}}, ad.alert(r, r.Alert), r.Kind)
}
for _, c := range vmExtras(src, r, now) {
add(c, ad.alert(r, r.Alert), r.Kind)
@@ -943,9 +949,18 @@ func judgeRecords(s wsnap, now time.Time, firing func(key string) bool, ad adopt
if sev == "" {
continue
}
add(Condition{Key: "workload/" + src + "/" + g.ns + "/" + g.owner, Severity: sev,
Title: fmt.Sprintf("%s (%s): %s", where(g.owner, g.ns), name, word), Href: workloadHref(src, g.ns, ownerName(g.owner))},
ad.alertKey(g.ns, g.ns+"/"+g.owner, g.alert), report.KindController)
c := Condition{Key: "workload/" + src + "/" + g.ns + "/" + g.owner, Severity: sev,
Title: fmt.Sprintf("%s (%s): %s", where(g.owner, g.ns), name, word), Href: workloadHref(src, g.ns, ownerName(g.owner))}
if g.bad > 0 && !strings.Contains(word, "; ") { // its pods failing, and nothing else: alike when they fail the same way
how := make([]string, 0, len(g.words))
for w := range g.words {
how = append(how, lowerFirst(w))
}
slices.Sort(how)
c.Batch = &Batch{Key: "workload/" + src + "/" + strings.Join(how, "+"), Title: "{n} workloads in " + name + " with pods failing: " + strings.Join(how, ", "),
Item: fmt.Sprintf("%s (%d of %d pods)", where(g.owner, g.ns), g.bad, g.pods), Href: workloadHref(src, "", "")}
}
add(c, ad.alertKey(g.ns, g.ns+"/"+g.owner, g.alert), report.KindController)
}
// Volumes: several failing in one storage class is one problem.
byClass := map[string][]pvcBad{}