From 8b2591982ea7820ee078aa61f649eecd84413e6e Mon Sep 17 00:00:00 2001 From: John Coffey Date: Sun, 15 Nov 2026 22:41:18 -0700 Subject: [PATCH] Alerts: alike problems from one place are one incident Twenty certificates from one Vault mount ending the same week were twenty pages. Now, after grouping by cause, three or more alike from one place fold into one incident that names them, worst first (eight, then a count), at the worst one's severity: - certificates ending, by issuer (the Certificates page's own), by Vault mount or cluster namespace, and Akamai CPS's; - VMs in one cluster in the same state; - workloads in one cluster whose pods fail the same way. Grouping's safety rules hold: one that has paged on its own stays on its own, and a batch that has paged keeps its incident while any of it is left. Certificates that have ended, or are weak or served wrong too, are never batched. Documented in formats.md, Batches. Closes #33. --- docs/formats.md | 22 +++++++ internal/server/akamai.go | 3 +- internal/server/api.go | 2 + internal/server/batches.go | 110 ++++++++++++++++++++++++++++++++ internal/server/batches_test.go | 58 +++++++++++++++++ internal/server/certs.go | 5 ++ internal/server/grouping.go | 10 ++- internal/server/workloads.go | 27 ++++++-- 8 files changed, 228 insertions(+), 9 deletions(-) create mode 100644 internal/server/batches.go create mode 100644 internal/server/batches_test.go diff --git a/docs/formats.md b/docs/formats.md index 2e3c939..00aa2d9 100644 --- a/docs/formats.md +++ b/docs/formats.md @@ -821,6 +821,28 @@ links to where it is on the maps. What it holds back and why is kept with the incident, so every copy shows it. The dashboard groups the same way, and the service map names the likely cause on a failing service. +### Batches + +After grouping by cause, problems that are alike and from one place are +one incident when there are three or more at once: "4 certificates from +pki_int in vault-prod end soon", "3 VMs in vcenter-phx: inaccessible". +Its detail names them, worst first (eight at most, then "and 5 more"), +and it takes the worst one's severity. What's alike: + +| batch | what's in it | +|---|---| +| certificates ending, by issuer | the Certificates page's own alerts (hosts' ports and files), when ending is all that's wrong | +| certificates ending, by Vault mount or cluster namespace | Vault PKI leaves and issuers; cert-manager's certificates | +| Akamai certificates ending | CPS's deployed certificates | +| VMs, by cluster and state | a vCenter's, Nutanix's, Proxmox's or KubeVirt's VMs failing the same way | +| workloads, by cluster and how their pods fail | Deployments, StatefulSets and the like whose pods all fail the same way (CrashLoopBackOff, say) | + +The safety rules are grouping's: one that has already paged on its own +stays on its own, and a batch that has paged stays one incident while +any of it is left, so it neither resolves early nor opens again as +something else. Certificates that have ended, or are weak or served +wrong as well, are never batched: each is its own job. + ## expected.yaml: what should run where Per host, `running` and `stopped` name containers; `units` names systemd diff --git a/internal/server/akamai.go b/internal/server/akamai.go index 71aa498..0015e7e 100644 --- a/internal/server/akamai.go +++ b/internal/server/akamai.go @@ -82,7 +82,8 @@ func (srv *Server) akamaiConditions(now time.Time) []Condition { if st, word := certStatus(c, now, srv.certCritical()); st != "good" { out = append(out, Condition{Key: key + "/" + c.Fingerprint, Severity: st, Ends: c.NotAfter, Title: "Akamai certificate for " + e.CN + ": " + lowerFirst(word), - Detail: akamaiEnrollment(e) + ", " + c.Key + ". " + akamaiRenewal(e), Href: href}) + Detail: akamaiEnrollment(e) + ", " + c.Key + ". " + akamaiRenewal(e), Href: href, + Batch: &Batch{Key: "akamai/cps-ends", Title: "{n} Akamai certificates end soon", Item: e.CN + " (" + certLeft(c.NotAfter.Sub(now)) + ")", Href: href}}) } } if what := akamaiWaiting[e.Waiting]; what != "" { diff --git a/internal/server/api.go b/internal/server/api.go index 75420fe..7fbd1f9 100644 --- a/internal/server/api.go +++ b/internal/server/api.go @@ -220,6 +220,8 @@ type Condition struct { // Ends is when the certificate a condition is about ends, for one that // is: certSeverity keeps it a warning until it's close (#34). Ends time.Time `json:"-"` + // Batch folds it with others alike from the same place (batches.go). + Batch *Batch `json:"-"` } // view judges every host now: what the UI shows, and the conditions the diff --git a/internal/server/batches.go b/internal/server/batches.go new file mode 100644 index 0000000..af3abde --- /dev/null +++ b/internal/server/batches.go @@ -0,0 +1,110 @@ +package server + +import ( + "cmp" + "slices" + "sort" + "strconv" + "strings" + + "github.com/cairnobs/cairnobs/internal/notify" +) + +// Batches (#33): several problems of one kind from one place at once (the +// certificates of one Vault mount ending in the same week, a vCenter's +// VMs all failing the same way) are one incident that lists them, not one +// page each. Root-cause grouping (grouping.go) folds what depends on a +// failure; this folds what's alike. +// +// The safety rules are grouping's: a member that has already paged on +// its own stays on its own, so it neither vanishes nor reads as resolved; +// the batch takes its worst member's severity, so one that gets worse +// escalates it; and a batch that has paged keeps its incident while any +// member is left, rather than resolving and opening again as one alert. + +// Batch says which batch a condition belongs to. +type Batch struct { + Key string // what's alike and where: "certends/vault-prod/pki_int" + Title string // the batch's title, {n} for how many: "{n} certificates from pki_int in vault-prod end soon" + Item string // this one in the batch's list: "svc-102.prod.internal (ends in 5 days)" + Href string // where the batch's link goes (the first member's link without it) +} + +// batchMin is how many alike make a batch. +const batchMin = 3 + +// batchListed is how many members a batch's detail names. +const batchListed = 8 + +// foldBatches replaces each batch's members with one condition, keyed +// batch/. firing says whether an incident has paged. +func foldBatches(conds []Condition, firing func(key string) bool) []Condition { + members := map[string][]int{} + for i, c := range conds { + if c.Batch != nil && !firing(c.Key) { + members[c.Batch.Key] = append(members[c.Batch.Key], i) + } + } + if len(members) == 0 { + return conds + } + keys := make([]string, 0, len(members)) + for k := range members { + keys = append(keys, k) + } + slices.Sort(keys) + folded := make([]bool, len(conds)) + batch := map[int]Condition{} // by its first member's place in conds, so it takes that place + for _, k := range keys { + idx := members[k] + if len(idx) < batchMin && !firing("batch/"+k) { + continue + } + at := slices.Min(idx) // where the batch goes in the list + // Worst first, then by key: the same order from one evaluation to + // the next, so the detail doesn't shuffle. + sort.SliceStable(idx, func(a, b int) bool { + ca, cb := conds[idx[a]], conds[idx[b]] + if ra, rb := notify.Rank(ca.Severity), notify.Rank(cb.Severity); ra != rb { + return ra > rb + } + return ca.Key < cb.Key + }) + first := conds[idx[0]] + b := Condition{Key: "batch/" + k, Severity: first.Severity, Href: cmp.Or(first.Batch.Href, first.Href), Map: first.Map, Host: first.Host, Ends: first.Ends, + Title: strings.ReplaceAll(first.Batch.Title, "{n}", strconv.Itoa(len(idx)))} + var items []string + for _, i := range idx { + c := conds[i] + folded[i] = true + if c.Host != b.Host { + b.Host = "" // a maintenance window holds it only when it's all one host's + } + if !c.Ends.IsZero() && (b.Ends.IsZero() || c.Ends.Before(b.Ends)) { + b.Ends = c.Ends + } + if len(items) < batchListed { + items = append(items, c.Batch.Item) + } + } + b.Detail = strings.Join(items, ", ") + if more := len(idx) - len(items); more > 0 { + b.Detail += " and " + strconv.Itoa(more) + " more" + } + b.Detail += "." + if len(idx) == 1 { // what's left of a batch that paged: still its incident + b.Title, b.Detail = first.Title, first.Detail + } + batch[at] = b + } + out := make([]Condition, 0, len(conds)) + for i, c := range conds { + if b, ok := batch[i]; ok { + out = append(out, b) + } + if !folded[i] { + out = append(out, c) + } + } + return out +} diff --git a/internal/server/batches_test.go b/internal/server/batches_test.go new file mode 100644 index 0000000..793eb28 --- /dev/null +++ b/internal/server/batches_test.go @@ -0,0 +1,58 @@ +package server + +import ( + "strings" + "testing" +) + +func batched(key, sev, item string) Condition { + return Condition{Key: key, Severity: sev, Title: key + " ends soon", Href: "/" + key, + Batch: &Batch{Key: "certends/vault-prod/pki_int", Title: "{n} certificates from pki_int end soon", Item: item, Href: "/workloads"}} +} + +// Three or more alike are one condition, named once each; fewer stay as +// they are; one that has paged on its own stays on its own (#33). +func TestFoldBatches(t *testing.T) { + none := func(string) bool { return false } + other := Condition{Key: "host/node1", Severity: "critical", Title: "node1 is down"} + conds := []Condition{ + batched("c", "warning", "c (6 days)"), other, + batched("a", "warning", "a (5 days)"), + batched("b", "critical", "b (30 hours)"), + } + out := foldBatches(conds, none) + if len(out) != 2 || out[0].Key != "batch/certends/vault-prod/pki_int" || out[1].Key != "host/node1" { + t.Fatalf("%+v", out) + } + b := out[0] + if b.Severity != "critical" || b.Title != "3 certificates from pki_int end soon" || b.Detail != "b (30 hours), a (5 days), c (6 days)." || b.Href != "/workloads" { + t.Errorf("batch: %+v", b) + } + + if out := foldBatches(conds[:3], none); len(out) != 3 { + t.Errorf("two alike aren't a batch: %+v", out) + } + + // b paged on its own before the others came: it stays, and the rest + // are too few. + paged := func(k string) bool { return k == "b" } + if out := foldBatches(conds, paged); len(out) != 4 { + t.Errorf("a paged member stays on its own: %+v", out) + } + + // A batch that paged keeps its incident while one member is left. + batchPaged := func(k string) bool { return k == "batch/certends/vault-prod/pki_int" } + out = foldBatches([]Condition{batched("a", "warning", "a (5 days)")}, batchPaged) + if len(out) != 1 || out[0].Key != "batch/certends/vault-prod/pki_int" || out[0].Title != "a ends soon" { + t.Errorf("what's left of a paged batch: %+v", out) + } + + // More than batchListed members: the rest are counted. + var many []Condition + for i := range 11 { + many = append(many, batched(string(rune('a'+i)), "warning", string(rune('a'+i)))) + } + if out := foldBatches(many, none); len(out) != 1 || !strings.HasSuffix(out[0].Detail, "h and 3 more.") { + t.Errorf("many: %+v", out) + } +} diff --git a/internal/server/certs.go b/internal/server/certs.go index 74f5e35..404dc63 100644 --- a/internal/server/certs.go +++ b/internal/server/certs.go @@ -3,6 +3,7 @@ package server import ( "net" "net/http" + "net/url" "slices" "strconv" "strings" @@ -571,6 +572,10 @@ func (srv *Server) certConditions(now time.Time) []Condition { cond.Host = h // a maintenance window on the host holds it } } + if own && c.Weak == "" && len(c.NotReloaded) == 0 && !wrongOnHosts(c) && c.Issuer != "" && c.NotAfter.After(now) { + cond.Batch = &Batch{Key: "certends/issuer/" + c.Issuer, Title: "{n} certificates from " + c.Issuer + " end soon", + Item: c.Subject + " (" + certLeft(c.NotAfter.Sub(now)) + ")", Href: "/certificates?q=" + url.QueryEscape(c.Issuer)} + } conds = append(conds, cond) } return conds diff --git a/internal/server/grouping.go b/internal/server/grouping.go index 122bf16..d4cee99 100644 --- a/internal/server/grouping.go +++ b/internal/server/grouping.go @@ -323,12 +323,18 @@ func nodeOf(t *Topology, c Condition) string { return "" } -// groupConditions folds conditions caused by a deeper failure into the +// groupConditions folds conditions by cause, then folds what's alike into +// batches (batches.go). +func (srv *Server) groupConditions(conds []Condition, firing func(key string) bool) []Condition { + return foldBatches(srv.groupByCause(conds, firing), firing) +} + +// groupByCause folds conditions caused by a deeper failure into the // condition for that failure. firing says whether a key already has an // incident that people were told about: such a condition stays on its // own, so it neither vanishes nor reads as "resolved" because its cause // was spotted later. -func (srv *Server) groupConditions(conds []Condition, firing func(key string) bool) []Condition { +func (srv *Server) groupByCause(conds []Condition, firing func(key string) bool) []Condition { if len(conds) == 0 { return conds } diff --git a/internal/server/workloads.go b/internal/server/workloads.go index e6ee9d0..14bd5b3 100644 --- a/internal/server/workloads.go +++ b/internal/server/workloads.go @@ -856,8 +856,13 @@ func judgeRecords(s wsnap, now time.Time, firing func(key string) bool, ad adopt detail = append(detail, "Served on "+on+".") } ends, _ := time.Parse(time.RFC3339, fmt.Sprint(r.Facts["ends"])) - add(Condition{Key: "cert/" + src + "/" + r.ID, Severity: sev, Detail: strings.Join(detail, " "), Ends: ends, - Title: fmt.Sprintf("%s %s%s in %s: %s", what, r.Name, from, name, lowerFirst(r.Word)), Href: workloadHref(src, r.Namespace, r.Name)}, ad.alert(r, r.Alert), r.Kind) + c := Condition{Key: "cert/" + src + "/" + r.ID, Severity: sev, Detail: strings.Join(detail, " "), Ends: ends, + Title: fmt.Sprintf("%s %s%s in %s: %s", what, r.Name, from, name, lowerFirst(r.Word)), Href: workloadHref(src, r.Namespace, r.Name)} + if strings.HasPrefix(r.Word, "Ends in") && !strings.Contains(r.Word, ";") && !ends.IsZero() { // ending, and nothing else wrong: alike + c.Batch = &Batch{Key: "certends/" + src + "/" + r.Namespace, Title: "{n} certificates" + from + " in " + name + " end soon", Href: workloadHref(src, r.Namespace, ""), + Item: r.Name + " (" + certLeft(ends.Sub(now)) + ")"} + } + add(c, ad.alert(r, r.Alert), r.Kind) } case report.KindVM: sev, detail := "", "" @@ -876,7 +881,8 @@ func judgeRecords(s wsnap, now time.Time, firing func(key string) bool, ad adopt } if sev != "" { add(Condition{Key: "vm/" + src + "/" + r.Namespace + "/" + r.Name, Severity: sev, Detail: detail, - Title: fmt.Sprintf("VM %s (%s): %s", where(r.Name, r.Namespace), name, lowerFirst(r.Word)), Href: workloadHref(src, r.Namespace, r.Name)}, ad.alert(r, r.Alert), r.Kind) + Title: fmt.Sprintf("VM %s (%s): %s", where(r.Name, r.Namespace), name, lowerFirst(r.Word)), Href: workloadHref(src, r.Namespace, r.Name), + Batch: &Batch{Key: "vm/" + src + "/" + r.Word, Title: "{n} VMs in " + name + ": " + lowerFirst(r.Word), Item: where(r.Name, r.Namespace), Href: workloadHref(src, "", "")}}, ad.alert(r, r.Alert), r.Kind) } for _, c := range vmExtras(src, r, now) { add(c, ad.alert(r, r.Alert), r.Kind) @@ -943,9 +949,18 @@ func judgeRecords(s wsnap, now time.Time, firing func(key string) bool, ad adopt if sev == "" { continue } - add(Condition{Key: "workload/" + src + "/" + g.ns + "/" + g.owner, Severity: sev, - Title: fmt.Sprintf("%s (%s): %s", where(g.owner, g.ns), name, word), Href: workloadHref(src, g.ns, ownerName(g.owner))}, - ad.alertKey(g.ns, g.ns+"/"+g.owner, g.alert), report.KindController) + c := Condition{Key: "workload/" + src + "/" + g.ns + "/" + g.owner, Severity: sev, + Title: fmt.Sprintf("%s (%s): %s", where(g.owner, g.ns), name, word), Href: workloadHref(src, g.ns, ownerName(g.owner))} + if g.bad > 0 && !strings.Contains(word, "; ") { // its pods failing, and nothing else: alike when they fail the same way + how := make([]string, 0, len(g.words)) + for w := range g.words { + how = append(how, lowerFirst(w)) + } + slices.Sort(how) + c.Batch = &Batch{Key: "workload/" + src + "/" + strings.Join(how, "+"), Title: "{n} workloads in " + name + " with pods failing: " + strings.Join(how, ", "), + Item: fmt.Sprintf("%s (%d of %d pods)", where(g.owner, g.ns), g.bad, g.pods), Href: workloadHref(src, "", "")} + } + add(c, ad.alertKey(g.ns, g.ns+"/"+g.owner, g.alert), report.KindController) } // Volumes: several failing in one storage class is one problem. byClass := map[string][]pvcBad{} -- 2.54.0