Twenty certificates from one Vault mount ending the same week were twenty pages. Now, after grouping by cause, three or more alike from one place fold into one incident that names them, worst first (eight, then a count), at the worst one's severity: - certificates ending, by issuer (the Certificates page's own), by Vault mount or cluster namespace, and Akamai CPS's; - VMs in one cluster in the same state; - workloads in one cluster whose pods fail the same way. Grouping's safety rules hold: one that has paged on its own stays on its own, and a batch that has paged keeps its incident while any of it is left. Certificates that have ended, or are weak or served wrong too, are never batched. Documented in formats.md, Batches. Closes #33.
487 lines
13 KiB
Go
487 lines
13 KiB
Go
package server
|
|
|
|
import (
|
|
"cmp"
|
|
"fmt"
|
|
"path"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/cairnobs/cairnobs/internal/mapgraph"
|
|
"github.com/cairnobs/cairnobs/internal/units"
|
|
)
|
|
|
|
// Root-cause grouping: when something fails and
|
|
// something it depends on has failed too, only the deeper failure becomes
|
|
// an incident; the others are listed under it as affected.
|
|
//
|
|
// What depends on what comes from two places (docs/maps.md, Learned
|
|
// edges): topology.yaml's needs, and the service map's edges that are
|
|
// declared (topology.yaml, Kubernetes, confirmed on the page) or learned
|
|
// and steady. A learned edge only seen so far, or fading, never groups,
|
|
// and a rejected one is gone from the map. An incident grouped by a
|
|
// learned edge says which, so a wrong guess is out in the open, and the
|
|
// safety rules stay: the group takes its worst severity, and an alert
|
|
// already sent stays on its own.
|
|
|
|
// GroupedBy is a learned dependency that grouped an incident.
|
|
type GroupedBy struct {
|
|
From string `json:"from"` // service ids on the map
|
|
To string `json:"to"`
|
|
FromName string `json:"from_name"`
|
|
ToName string `json:"to_name"`
|
|
Port int `json:"port,omitempty"`
|
|
Since time.Time `json:"since"` // first seen
|
|
}
|
|
|
|
func (g GroupedBy) String() string {
|
|
p := ""
|
|
if g.Port != 0 {
|
|
p = fmt.Sprintf(" :%d", g.Port)
|
|
}
|
|
return fmt.Sprintf("%s has talked to %s%s since %s", g.FromName, g.ToName, p, g.Since.Format("Jan 2"))
|
|
}
|
|
|
|
// depEdge is one direct need; why is set when it's a learned edge.
|
|
type depEdge struct {
|
|
to string
|
|
why *GroupedBy
|
|
}
|
|
|
|
// deps is what grouping works from: nodes are topology.yaml's services,
|
|
// the map's services and hosts.
|
|
type deps struct {
|
|
topo *Topology
|
|
needs map[string][]depEdge
|
|
names map[string]string
|
|
parts map[string]string // mapgraph.PartKey → the map service it's part of
|
|
svc map[string]bool // map services, for a link to the service map
|
|
hosts map[string]bool
|
|
}
|
|
|
|
func (d *deps) add(from, to string, why *GroupedBy) {
|
|
if from == "" || to == "" || from == to {
|
|
return
|
|
}
|
|
for i, e := range d.needs[from] {
|
|
if e.to == to {
|
|
if e.why != nil && why == nil { // declared beats learned
|
|
d.needs[from][i].why = nil
|
|
}
|
|
return
|
|
}
|
|
}
|
|
d.needs[from] = append(d.needs[from], depEdge{to, why})
|
|
}
|
|
|
|
// dependencies gathers topology.yaml's and the map's dependencies; nil
|
|
// when there are neither.
|
|
func (srv *Server) dependencies(now time.Time) *deps {
|
|
t := srv.currentTopology()
|
|
d := &deps{topo: t, needs: map[string][]depEdge{}, names: map[string]string{}, parts: map[string]string{},
|
|
svc: map[string]bool{}, hosts: map[string]bool{}}
|
|
if t != nil {
|
|
for _, s := range t.Services {
|
|
d.names[s.ID] = s.Name
|
|
for _, n := range s.Needs {
|
|
d.add(s.ID, n, nil)
|
|
}
|
|
}
|
|
}
|
|
g := srv.mapGraph(now)
|
|
for _, h := range g.Hosts {
|
|
d.hosts[h.Name] = true
|
|
}
|
|
for _, s := range g.Services {
|
|
d.svc[s.ID] = true
|
|
if d.names[s.ID] == "" {
|
|
d.names[s.ID] = s.Name
|
|
}
|
|
var on []string
|
|
for _, p := range s.Parts {
|
|
d.parts[mapgraph.PartKey(p.Host, p.Owner)] = s.ID
|
|
on = appendNew(on, p.Host)
|
|
}
|
|
if len(on) == 1 && s.Source != "kubernetes" { // all of it on one host: that host going takes it down
|
|
d.add(s.ID, on[0], nil)
|
|
}
|
|
}
|
|
for _, e := range srv.mapEdges(g, now) {
|
|
switch e.Stage {
|
|
case stageDeclared:
|
|
d.add(e.From, e.To, nil)
|
|
case stageSteady:
|
|
why := &GroupedBy{From: e.From, To: e.To, FromName: d.names[e.From], ToName: d.names[e.To], Port: e.Port}
|
|
if e.First != nil {
|
|
why.Since = *e.First
|
|
}
|
|
d.add(e.From, e.To, why)
|
|
}
|
|
}
|
|
// A host that's a VM needs that VM, and the VM its node (vmlink.go):
|
|
// keyed as their conditions are.
|
|
byHost, _ := srv.vmLinks()
|
|
for h, v := range byHost {
|
|
d.add(h, v.key, nil)
|
|
d.names[v.key] = "VM " + where(v.Name, v.Namespace) + " (" + v.place() + ")"
|
|
if v.Node != "" {
|
|
n := "node/" + v.Source + "/" + v.Node
|
|
d.add(v.key, n, nil)
|
|
d.names[n] = "node " + v.Node + " (" + v.Source + ")"
|
|
}
|
|
}
|
|
// A libvirt VM needs the host it runs on: that host silent explains it,
|
|
// and through it the guest's own agent going quiet.
|
|
for _, h := range srv.store.HostVMs() {
|
|
for _, r := range h.vms.List {
|
|
k := "vm/" + h.name + "//" + r.Name
|
|
d.add(k, h.name, nil)
|
|
d.names[k] = "VM " + r.Name + " (" + h.name + ")"
|
|
}
|
|
}
|
|
if t == nil && len(d.needs) == 0 {
|
|
return nil
|
|
}
|
|
return d
|
|
}
|
|
|
|
// nodeOf says which node a condition belongs to: a service, a host, or ""
|
|
// when nothing covers it (it then stands alone).
|
|
func (d *deps) nodeOf(c Condition) string {
|
|
n := ""
|
|
if d.topo != nil {
|
|
n = nodeOf(d.topo, c)
|
|
}
|
|
parts := strings.SplitN(c.Key, "/", 3)
|
|
switch parts[0] {
|
|
case "silent":
|
|
return c.Host
|
|
case "vm", "node": // a cluster's VM or node, as dependencies keys them
|
|
return c.Key
|
|
case "unit", "container":
|
|
if len(parts) < 3 {
|
|
return n
|
|
}
|
|
host := parts[1]
|
|
if n != "" && n != host {
|
|
return n // topology.yaml placed it
|
|
}
|
|
if s := d.parts[mapgraph.PartKey(host, parts[0]+":"+parts[2])]; s != "" {
|
|
return s
|
|
}
|
|
return host
|
|
case "workload": // workload/<cluster>/<ns>/<controller>: its service on the map
|
|
if len(parts) == 3 {
|
|
if s := d.parts[mapgraph.PartKey(parts[1], "k8s:"+parts[2])]; s != "" {
|
|
return s
|
|
}
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
// mapHref is where a node is on the maps, or "".
|
|
func (d *deps) mapHref(node string) string {
|
|
switch {
|
|
case d.svc[node]:
|
|
return "/map/services?focus=svc:" + node
|
|
case d.hosts[node]:
|
|
return "/map/network?focus=host:" + node
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// path is a shortest way from one node to another through what each
|
|
// needs, as the edges taken; nil when there's none.
|
|
func (d *deps) path(from, to string) []depEdge {
|
|
type step struct {
|
|
prev string
|
|
e depEdge
|
|
}
|
|
seen := map[string]step{from: {}}
|
|
queue := []string{from}
|
|
for len(queue) > 0 {
|
|
n := queue[0]
|
|
queue = queue[1:]
|
|
if n == to {
|
|
var out []depEdge
|
|
for n != from {
|
|
s := seen[n]
|
|
out = append(out, s.e)
|
|
n = s.prev
|
|
}
|
|
slices.Reverse(out)
|
|
return out
|
|
}
|
|
for _, e := range d.needs[n] {
|
|
if _, ok := seen[e.to]; !ok {
|
|
seen[e.to] = step{n, e}
|
|
queue = append(queue, e.to)
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// reach is everything a node depends on, all the way down.
|
|
func (d *deps) reach(n string) map[string]bool {
|
|
seen := map[string]bool{}
|
|
todo := []string{n}
|
|
for len(todo) > 0 {
|
|
x := todo[len(todo)-1]
|
|
todo = todo[:len(todo)-1]
|
|
for _, e := range d.needs[x] {
|
|
if !seen[e.to] {
|
|
seen[e.to] = true
|
|
todo = append(todo, e.to)
|
|
}
|
|
}
|
|
}
|
|
return seen
|
|
}
|
|
|
|
// nodeOf says which topology.yaml node a condition belongs to: a service,
|
|
// a host, or "" when the file doesn't cover it.
|
|
func nodeOf(t *Topology, c Condition) string {
|
|
parts := strings.SplitN(c.Key, "/", 3)
|
|
kind := parts[0]
|
|
switch kind {
|
|
case "silent":
|
|
return c.Host // a host that's gone explains everything on it
|
|
case "disk", "memory", "containers-unreadable":
|
|
return "" // trouble on a host, but not why its services fail: stands alone
|
|
case "unit", "container":
|
|
if len(parts) < 3 {
|
|
return ""
|
|
}
|
|
host, name := parts[1], parts[2]
|
|
for _, s := range t.Services {
|
|
pats := s.Containers
|
|
if kind == "unit" {
|
|
pats = nil
|
|
for _, u := range s.Units {
|
|
h, n, _ := strings.Cut(u, "/")
|
|
pats = append(pats, h+"/"+units.Name(n))
|
|
}
|
|
}
|
|
for _, p := range pats {
|
|
if hostMatch(p, host, name) {
|
|
return s.ID
|
|
}
|
|
}
|
|
}
|
|
return host // not in any service: at least it's on that host
|
|
case "check", "cert":
|
|
id := strings.TrimPrefix(c.Key, kind+"/")
|
|
for _, s := range t.Services {
|
|
if slices.Contains(s.Checks, id) {
|
|
return s.ID
|
|
}
|
|
}
|
|
case "drift": // drift/<container>: the service that runs it
|
|
name := strings.TrimPrefix(c.Key, "drift/")
|
|
for _, s := range t.Services {
|
|
for _, p := range s.Containers {
|
|
if _, n, _ := strings.Cut(p, "/"); func() bool { ok, _ := path.Match(n, name); return ok }() {
|
|
return s.ID
|
|
}
|
|
}
|
|
}
|
|
return ""
|
|
case "dl": // data layer: dl/<service>/...; the service map ids match (patroni, etcd, haproxy, nats, garage)
|
|
if len(parts) > 1 {
|
|
for _, s := range t.Services {
|
|
if s.ID == parts[1] {
|
|
return s.ID
|
|
}
|
|
}
|
|
}
|
|
return c.Host
|
|
case "mail": // the monitor account's view of mail: the service map's mail node
|
|
for _, s := range t.Services {
|
|
if s.ID == "mail" {
|
|
return s.ID
|
|
}
|
|
}
|
|
return ""
|
|
case "akamai":
|
|
if len(parts) > 1 && parts[1] == "server" { // GTM can't reach a host: that's the host's trouble
|
|
return c.Host
|
|
}
|
|
case "cloudflare":
|
|
if len(parts) > 1 && parts[1] == "origin" { // Cloudflare can't reach a host: that's the host's trouble
|
|
return c.Host
|
|
}
|
|
for _, s := range t.Services {
|
|
if s.Cloudflare {
|
|
return s.ID
|
|
}
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// groupConditions folds conditions by cause, then folds what's alike into
|
|
// batches (batches.go).
|
|
func (srv *Server) groupConditions(conds []Condition, firing func(key string) bool) []Condition {
|
|
return foldBatches(srv.groupByCause(conds, firing), firing)
|
|
}
|
|
|
|
// groupByCause folds conditions caused by a deeper failure into the
|
|
// condition for that failure. firing says whether a key already has an
|
|
// incident that people were told about: such a condition stays on its
|
|
// own, so it neither vanishes nor reads as "resolved" because its cause
|
|
// was spotted later.
|
|
func (srv *Server) groupByCause(conds []Condition, firing func(key string) bool) []Condition {
|
|
if len(conds) == 0 {
|
|
return conds
|
|
}
|
|
d := srv.dependencies(time.Now())
|
|
if d == nil {
|
|
return conds
|
|
}
|
|
node := make([]string, len(conds))
|
|
failing := map[string]bool{}
|
|
for i, c := range conds {
|
|
node[i] = d.nodeOf(c)
|
|
if node[i] != "" {
|
|
failing[node[i]] = true
|
|
}
|
|
if conds[i].Map == "" {
|
|
conds[i].Map = d.mapHref(node[i])
|
|
}
|
|
}
|
|
if len(conds) < 2 {
|
|
return conds
|
|
}
|
|
// What each failing node depends on, worked out only for those.
|
|
below := map[string]map[string]bool{}
|
|
for n := range failing {
|
|
below[n] = d.reach(n)
|
|
}
|
|
// A failing node is a root when nothing it needs is failing.
|
|
isRoot := func(n string) bool {
|
|
for m := range below[n] {
|
|
if failing[m] {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
// The condition that speaks for each root node: the most severe, then
|
|
// the first by key, so the choice is stable from one evaluation to the next.
|
|
order := make([]int, len(conds))
|
|
for i := range order {
|
|
order[i] = i
|
|
}
|
|
sort.SliceStable(order, func(a, b int) bool {
|
|
ca, cb := conds[order[a]], conds[order[b]]
|
|
if ca.Severity != cb.Severity {
|
|
return ca.Severity == "critical"
|
|
}
|
|
return ca.Key < cb.Key
|
|
})
|
|
speaker := map[string]int{}
|
|
for _, i := range order {
|
|
if n := node[i]; n != "" && isRoot(n) {
|
|
if _, ok := speaker[n]; !ok {
|
|
speaker[n] = i
|
|
}
|
|
}
|
|
}
|
|
|
|
type group struct {
|
|
affected []string // node names, in order of appearance
|
|
held int
|
|
severity string
|
|
why []GroupedBy
|
|
}
|
|
groups := map[int]*group{}
|
|
keep := make([]bool, len(conds))
|
|
for _, i := range order {
|
|
c, n := conds[i], node[i]
|
|
keep[i] = true
|
|
if n == "" || isRoot(n) || firing(c.Key) {
|
|
continue
|
|
}
|
|
// Hand it to the first root (in speaker order) that it depends on.
|
|
root := -1
|
|
for _, j := range order {
|
|
if m := node[j]; m != "" && speaker[m] == j && below[n][m] {
|
|
root = j
|
|
break
|
|
}
|
|
}
|
|
if root < 0 {
|
|
continue
|
|
}
|
|
keep[i] = false
|
|
g := groups[root]
|
|
if g == nil {
|
|
g = &group{severity: conds[root].Severity}
|
|
groups[root] = g
|
|
}
|
|
g.held++
|
|
label := cmp.Or(d.names[n], n)
|
|
if !slices.Contains(g.affected, label) {
|
|
g.affected = append(g.affected, label)
|
|
}
|
|
if c.Severity == "critical" {
|
|
g.severity = "critical"
|
|
}
|
|
for _, e := range d.path(n, node[root]) {
|
|
if e.why != nil && !slices.Contains(g.why, *e.why) {
|
|
g.why = append(g.why, *e.why)
|
|
}
|
|
}
|
|
}
|
|
out := make([]Condition, 0, len(conds))
|
|
for i, c := range conds {
|
|
if !keep[i] {
|
|
continue
|
|
}
|
|
if g := groups[i]; g != nil {
|
|
c.Severity = g.severity
|
|
c.Affected = strings.Join(g.affected, ", ")
|
|
c.Grouped = g.why
|
|
d := strings.TrimSpace(c.Detail)
|
|
if d != "" && !strings.HasSuffix(d, ".") {
|
|
d += "."
|
|
}
|
|
d += fmt.Sprintf(" Also affected: %s (%s held back).", c.Affected, plural(g.held, "alert", "alerts"))
|
|
if len(g.why) > 0 {
|
|
var ws []string
|
|
for _, w := range g.why[:min(len(g.why), 3)] {
|
|
ws = append(ws, w.String())
|
|
}
|
|
more := ""
|
|
if len(g.why) > 3 {
|
|
more = fmt.Sprintf(" (and %d more)", len(g.why)-3)
|
|
}
|
|
d += " Grouped because " + strings.Join(ws, "; ") + more + "."
|
|
}
|
|
c.Detail = strings.TrimSpace(d)
|
|
}
|
|
out = append(out, c)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// needsClosure is every topology.yaml node each node depends on, all the
|
|
// way down, for the hand-drawn service map.
|
|
func needsClosure(t *Topology) map[string]map[string]bool {
|
|
d := &deps{needs: map[string][]depEdge{}}
|
|
for _, s := range t.Services {
|
|
for _, n := range s.Needs {
|
|
d.add(s.ID, n, nil)
|
|
}
|
|
}
|
|
out := map[string]map[string]bool{}
|
|
for _, s := range t.Services {
|
|
out[s.ID] = d.reach(s.ID)
|
|
}
|
|
return out
|
|
}
|