Alerts: alike problems from one place are one incident #50
No files matched your search
@@ -821,6 +821,28 @@ links to where it is on the maps. What it holds back and why is kept
|
||||
with the incident, so every copy shows it. The dashboard groups the same
|
||||
way, and the service map names the likely cause on a failing service.
|
||||
|
||||
### Batches
|
||||
|
||||
After grouping by cause, problems that are alike and from one place are
|
||||
one incident when there are three or more at once: "4 certificates from
|
||||
pki_int in vault-prod end soon", "3 VMs in vcenter-phx: inaccessible".
|
||||
Its detail names them, worst first (eight at most, then "and 5 more"),
|
||||
and it takes the worst one's severity. What's alike:
|
||||
|
||||
| batch | what's in it |
|
||||
|---|---|
|
||||
| certificates ending, by issuer | the Certificates page's own alerts (hosts' ports and files), when ending is all that's wrong |
|
||||
| certificates ending, by Vault mount or cluster namespace | Vault PKI leaves and issuers; cert-manager's certificates |
|
||||
| Akamai certificates ending | CPS's deployed certificates |
|
||||
| VMs, by cluster and state | a vCenter's, Nutanix's, Proxmox's or KubeVirt's VMs failing the same way |
|
||||
| workloads, by cluster and how their pods fail | Deployments, StatefulSets and the like whose pods all fail the same way (CrashLoopBackOff, say) |
|
||||
|
||||
The safety rules are grouping's: one that has already paged on its own
|
||||
stays on its own, and a batch that has paged stays one incident while
|
||||
any of it is left, so it neither resolves early nor opens again as
|
||||
something else. Certificates that have ended, or are weak or served
|
||||
wrong as well, are never batched: each is its own job.
|
||||
|
||||
## expected.yaml: what should run where
|
||||
|
||||
Per host, `running` and `stopped` name containers; `units` names systemd
|
||||
|
||||
@@ -82,7 +82,8 @@ func (srv *Server) akamaiConditions(now time.Time) []Condition {
|
||||
if st, word := certStatus(c, now, srv.certCritical()); st != "good" {
|
||||
out = append(out, Condition{Key: key + "/" + c.Fingerprint, Severity: st, Ends: c.NotAfter,
|
||||
Title: "Akamai certificate for " + e.CN + ": " + lowerFirst(word),
|
||||
Detail: akamaiEnrollment(e) + ", " + c.Key + ". " + akamaiRenewal(e), Href: href})
|
||||
Detail: akamaiEnrollment(e) + ", " + c.Key + ". " + akamaiRenewal(e), Href: href,
|
||||
Batch: &Batch{Key: "akamai/cps-ends", Title: "{n} Akamai certificates end soon", Item: e.CN + " (" + certLeft(c.NotAfter.Sub(now)) + ")", Href: href}})
|
||||
}
|
||||
}
|
||||
if what := akamaiWaiting[e.Waiting]; what != "" {
|
||||
|
||||
@@ -220,6 +220,8 @@ type Condition struct {
|
||||
// Ends is when the certificate a condition is about ends, for one that
|
||||
// is: certSeverity keeps it a warning until it's close (#34).
|
||||
Ends time.Time `json:"-"`
|
||||
// Batch folds it with others alike from the same place (batches.go).
|
||||
Batch *Batch `json:"-"`
|
||||
}
|
||||
|
||||
// view judges every host now: what the UI shows, and the conditions the
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/cairnobs/cairnobs/internal/notify"
|
||||
)
|
||||
|
||||
// Batches (#33): several problems of one kind from one place at once (the
|
||||
// certificates of one Vault mount ending in the same week, a vCenter's
|
||||
// VMs all failing the same way) are one incident that lists them, not one
|
||||
// page each. Root-cause grouping (grouping.go) folds what depends on a
|
||||
// failure; this folds what's alike.
|
||||
//
|
||||
// The safety rules are grouping's: a member that has already paged on
|
||||
// its own stays on its own, so it neither vanishes nor reads as resolved;
|
||||
// the batch takes its worst member's severity, so one that gets worse
|
||||
// escalates it; and a batch that has paged keeps its incident while any
|
||||
// member is left, rather than resolving and opening again as one alert.
|
||||
|
||||
// Batch says which batch a condition belongs to.
|
||||
type Batch struct {
|
||||
Key string // what's alike and where: "certends/vault-prod/pki_int"
|
||||
Title string // the batch's title, {n} for how many: "{n} certificates from pki_int in vault-prod end soon"
|
||||
Item string // this one in the batch's list: "svc-102.prod.internal (ends in 5 days)"
|
||||
Href string // where the batch's link goes (the first member's link without it)
|
||||
}
|
||||
|
||||
// batchMin is how many alike make a batch.
|
||||
const batchMin = 3
|
||||
|
||||
// batchListed is how many members a batch's detail names.
|
||||
const batchListed = 8
|
||||
|
||||
// foldBatches replaces each batch's members with one condition, keyed
|
||||
// batch/<key>. firing says whether an incident has paged.
|
||||
func foldBatches(conds []Condition, firing func(key string) bool) []Condition {
|
||||
members := map[string][]int{}
|
||||
for i, c := range conds {
|
||||
if c.Batch != nil && !firing(c.Key) {
|
||||
members[c.Batch.Key] = append(members[c.Batch.Key], i)
|
||||
}
|
||||
}
|
||||
if len(members) == 0 {
|
||||
return conds
|
||||
}
|
||||
keys := make([]string, 0, len(members))
|
||||
for k := range members {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
slices.Sort(keys)
|
||||
folded := make([]bool, len(conds))
|
||||
batch := map[int]Condition{} // by its first member's place in conds, so it takes that place
|
||||
for _, k := range keys {
|
||||
idx := members[k]
|
||||
if len(idx) < batchMin && !firing("batch/"+k) {
|
||||
continue
|
||||
}
|
||||
at := slices.Min(idx) // where the batch goes in the list
|
||||
// Worst first, then by key: the same order from one evaluation to
|
||||
// the next, so the detail doesn't shuffle.
|
||||
sort.SliceStable(idx, func(a, b int) bool {
|
||||
ca, cb := conds[idx[a]], conds[idx[b]]
|
||||
if ra, rb := notify.Rank(ca.Severity), notify.Rank(cb.Severity); ra != rb {
|
||||
return ra > rb
|
||||
}
|
||||
return ca.Key < cb.Key
|
||||
})
|
||||
first := conds[idx[0]]
|
||||
b := Condition{Key: "batch/" + k, Severity: first.Severity, Href: cmp.Or(first.Batch.Href, first.Href), Map: first.Map, Host: first.Host, Ends: first.Ends,
|
||||
Title: strings.ReplaceAll(first.Batch.Title, "{n}", strconv.Itoa(len(idx)))}
|
||||
var items []string
|
||||
for _, i := range idx {
|
||||
c := conds[i]
|
||||
folded[i] = true
|
||||
if c.Host != b.Host {
|
||||
b.Host = "" // a maintenance window holds it only when it's all one host's
|
||||
}
|
||||
if !c.Ends.IsZero() && (b.Ends.IsZero() || c.Ends.Before(b.Ends)) {
|
||||
b.Ends = c.Ends
|
||||
}
|
||||
if len(items) < batchListed {
|
||||
items = append(items, c.Batch.Item)
|
||||
}
|
||||
}
|
||||
b.Detail = strings.Join(items, ", ")
|
||||
if more := len(idx) - len(items); more > 0 {
|
||||
b.Detail += " and " + strconv.Itoa(more) + " more"
|
||||
}
|
||||
b.Detail += "."
|
||||
if len(idx) == 1 { // what's left of a batch that paged: still its incident
|
||||
b.Title, b.Detail = first.Title, first.Detail
|
||||
}
|
||||
batch[at] = b
|
||||
}
|
||||
out := make([]Condition, 0, len(conds))
|
||||
for i, c := range conds {
|
||||
if b, ok := batch[i]; ok {
|
||||
out = append(out, b)
|
||||
}
|
||||
if !folded[i] {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func batched(key, sev, item string) Condition {
|
||||
return Condition{Key: key, Severity: sev, Title: key + " ends soon", Href: "/" + key,
|
||||
Batch: &Batch{Key: "certends/vault-prod/pki_int", Title: "{n} certificates from pki_int end soon", Item: item, Href: "/workloads"}}
|
||||
}
|
||||
|
||||
// Three or more alike are one condition, named once each; fewer stay as
|
||||
// they are; one that has paged on its own stays on its own (#33).
|
||||
func TestFoldBatches(t *testing.T) {
|
||||
none := func(string) bool { return false }
|
||||
other := Condition{Key: "host/node1", Severity: "critical", Title: "node1 is down"}
|
||||
conds := []Condition{
|
||||
batched("c", "warning", "c (6 days)"), other,
|
||||
batched("a", "warning", "a (5 days)"),
|
||||
batched("b", "critical", "b (30 hours)"),
|
||||
}
|
||||
out := foldBatches(conds, none)
|
||||
if len(out) != 2 || out[0].Key != "batch/certends/vault-prod/pki_int" || out[1].Key != "host/node1" {
|
||||
t.Fatalf("%+v", out)
|
||||
}
|
||||
b := out[0]
|
||||
if b.Severity != "critical" || b.Title != "3 certificates from pki_int end soon" || b.Detail != "b (30 hours), a (5 days), c (6 days)." || b.Href != "/workloads" {
|
||||
t.Errorf("batch: %+v", b)
|
||||
}
|
||||
|
||||
if out := foldBatches(conds[:3], none); len(out) != 3 {
|
||||
t.Errorf("two alike aren't a batch: %+v", out)
|
||||
}
|
||||
|
||||
// b paged on its own before the others came: it stays, and the rest
|
||||
// are too few.
|
||||
paged := func(k string) bool { return k == "b" }
|
||||
if out := foldBatches(conds, paged); len(out) != 4 {
|
||||
t.Errorf("a paged member stays on its own: %+v", out)
|
||||
}
|
||||
|
||||
// A batch that paged keeps its incident while one member is left.
|
||||
batchPaged := func(k string) bool { return k == "batch/certends/vault-prod/pki_int" }
|
||||
out = foldBatches([]Condition{batched("a", "warning", "a (5 days)")}, batchPaged)
|
||||
if len(out) != 1 || out[0].Key != "batch/certends/vault-prod/pki_int" || out[0].Title != "a ends soon" {
|
||||
t.Errorf("what's left of a paged batch: %+v", out)
|
||||
}
|
||||
|
||||
// More than batchListed members: the rest are counted.
|
||||
var many []Condition
|
||||
for i := range 11 {
|
||||
many = append(many, batched(string(rune('a'+i)), "warning", string(rune('a'+i))))
|
||||
}
|
||||
if out := foldBatches(many, none); len(out) != 1 || !strings.HasSuffix(out[0].Detail, "h and 3 more.") {
|
||||
t.Errorf("many: %+v", out)
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package server
|
||||
import (
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -571,6 +572,10 @@ func (srv *Server) certConditions(now time.Time) []Condition {
|
||||
cond.Host = h // a maintenance window on the host holds it
|
||||
}
|
||||
}
|
||||
if own && c.Weak == "" && len(c.NotReloaded) == 0 && !wrongOnHosts(c) && c.Issuer != "" && c.NotAfter.After(now) {
|
||||
cond.Batch = &Batch{Key: "certends/issuer/" + c.Issuer, Title: "{n} certificates from " + c.Issuer + " end soon",
|
||||
Item: c.Subject + " (" + certLeft(c.NotAfter.Sub(now)) + ")", Href: "/certificates?q=" + url.QueryEscape(c.Issuer)}
|
||||
}
|
||||
conds = append(conds, cond)
|
||||
}
|
||||
return conds
|
||||
|
||||
@@ -323,12 +323,18 @@ func nodeOf(t *Topology, c Condition) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// groupConditions folds conditions caused by a deeper failure into the
|
||||
// groupConditions folds conditions by cause, then folds what's alike into
|
||||
// batches (batches.go).
|
||||
func (srv *Server) groupConditions(conds []Condition, firing func(key string) bool) []Condition {
|
||||
return foldBatches(srv.groupByCause(conds, firing), firing)
|
||||
}
|
||||
|
||||
// groupByCause folds conditions caused by a deeper failure into the
|
||||
// condition for that failure. firing says whether a key already has an
|
||||
// incident that people were told about: such a condition stays on its
|
||||
// own, so it neither vanishes nor reads as "resolved" because its cause
|
||||
// was spotted later.
|
||||
func (srv *Server) groupConditions(conds []Condition, firing func(key string) bool) []Condition {
|
||||
func (srv *Server) groupByCause(conds []Condition, firing func(key string) bool) []Condition {
|
||||
if len(conds) == 0 {
|
||||
return conds
|
||||
}
|
||||
|
||||
@@ -856,8 +856,13 @@ func judgeRecords(s wsnap, now time.Time, firing func(key string) bool, ad adopt
|
||||
detail = append(detail, "Served on "+on+".")
|
||||
}
|
||||
ends, _ := time.Parse(time.RFC3339, fmt.Sprint(r.Facts["ends"]))
|
||||
add(Condition{Key: "cert/" + src + "/" + r.ID, Severity: sev, Detail: strings.Join(detail, " "), Ends: ends,
|
||||
Title: fmt.Sprintf("%s %s%s in %s: %s", what, r.Name, from, name, lowerFirst(r.Word)), Href: workloadHref(src, r.Namespace, r.Name)}, ad.alert(r, r.Alert), r.Kind)
|
||||
c := Condition{Key: "cert/" + src + "/" + r.ID, Severity: sev, Detail: strings.Join(detail, " "), Ends: ends,
|
||||
Title: fmt.Sprintf("%s %s%s in %s: %s", what, r.Name, from, name, lowerFirst(r.Word)), Href: workloadHref(src, r.Namespace, r.Name)}
|
||||
if strings.HasPrefix(r.Word, "Ends in") && !strings.Contains(r.Word, ";") && !ends.IsZero() { // ending, and nothing else wrong: alike
|
||||
c.Batch = &Batch{Key: "certends/" + src + "/" + r.Namespace, Title: "{n} certificates" + from + " in " + name + " end soon", Href: workloadHref(src, r.Namespace, ""),
|
||||
Item: r.Name + " (" + certLeft(ends.Sub(now)) + ")"}
|
||||
}
|
||||
add(c, ad.alert(r, r.Alert), r.Kind)
|
||||
}
|
||||
case report.KindVM:
|
||||
sev, detail := "", ""
|
||||
@@ -876,7 +881,8 @@ func judgeRecords(s wsnap, now time.Time, firing func(key string) bool, ad adopt
|
||||
}
|
||||
if sev != "" {
|
||||
add(Condition{Key: "vm/" + src + "/" + r.Namespace + "/" + r.Name, Severity: sev, Detail: detail,
|
||||
Title: fmt.Sprintf("VM %s (%s): %s", where(r.Name, r.Namespace), name, lowerFirst(r.Word)), Href: workloadHref(src, r.Namespace, r.Name)}, ad.alert(r, r.Alert), r.Kind)
|
||||
Title: fmt.Sprintf("VM %s (%s): %s", where(r.Name, r.Namespace), name, lowerFirst(r.Word)), Href: workloadHref(src, r.Namespace, r.Name),
|
||||
Batch: &Batch{Key: "vm/" + src + "/" + r.Word, Title: "{n} VMs in " + name + ": " + lowerFirst(r.Word), Item: where(r.Name, r.Namespace), Href: workloadHref(src, "", "")}}, ad.alert(r, r.Alert), r.Kind)
|
||||
}
|
||||
for _, c := range vmExtras(src, r, now) {
|
||||
add(c, ad.alert(r, r.Alert), r.Kind)
|
||||
@@ -943,9 +949,18 @@ func judgeRecords(s wsnap, now time.Time, firing func(key string) bool, ad adopt
|
||||
if sev == "" {
|
||||
continue
|
||||
}
|
||||
add(Condition{Key: "workload/" + src + "/" + g.ns + "/" + g.owner, Severity: sev,
|
||||
Title: fmt.Sprintf("%s (%s): %s", where(g.owner, g.ns), name, word), Href: workloadHref(src, g.ns, ownerName(g.owner))},
|
||||
ad.alertKey(g.ns, g.ns+"/"+g.owner, g.alert), report.KindController)
|
||||
c := Condition{Key: "workload/" + src + "/" + g.ns + "/" + g.owner, Severity: sev,
|
||||
Title: fmt.Sprintf("%s (%s): %s", where(g.owner, g.ns), name, word), Href: workloadHref(src, g.ns, ownerName(g.owner))}
|
||||
if g.bad > 0 && !strings.Contains(word, "; ") { // its pods failing, and nothing else: alike when they fail the same way
|
||||
how := make([]string, 0, len(g.words))
|
||||
for w := range g.words {
|
||||
how = append(how, lowerFirst(w))
|
||||
}
|
||||
slices.Sort(how)
|
||||
c.Batch = &Batch{Key: "workload/" + src + "/" + strings.Join(how, "+"), Title: "{n} workloads in " + name + " with pods failing: " + strings.Join(how, ", "),
|
||||
Item: fmt.Sprintf("%s (%d of %d pods)", where(g.owner, g.ns), g.bad, g.pods), Href: workloadHref(src, "", "")}
|
||||
}
|
||||
add(c, ad.alertKey(g.ns, g.ns+"/"+g.owner, g.alert), report.KindController)
|
||||
}
|
||||
// Volumes: several failing in one storage class is one problem.
|
||||
byClass := map[string][]pvcBad{}
|
||||
|
||||
Reference in new issue
Block a user