Files
cairnobs/internal/server/grouping.go
T
jcoffey-dev 8b2591982e
ci / check (pull_request) Successful in 7m1s
ci / postgres (pull_request) Successful in 10m26s
Alerts: alike problems from one place are one incident
Twenty certificates from one Vault mount ending the same week were twenty
pages. Now, after grouping by cause, three or more alike from one place
fold into one incident that names them, worst first (eight, then a
count), at the worst one's severity:

- certificates ending, by issuer (the Certificates page's own), by Vault
  mount or cluster namespace, and Akamai CPS's;
- VMs in one cluster in the same state;
- workloads in one cluster whose pods fail the same way.

Grouping's safety rules hold: one that has paged on its own stays on its
own, and a batch that has paged keeps its incident while any of it is
left. Certificates that have ended, or are weak or served wrong too, are
never batched. Documented in formats.md, Batches.

Closes #33.
2026-11-15 22:52:37 -07:00

487 lines
13 KiB
Go

package server
import (
"cmp"
"fmt"
"path"
"slices"
"sort"
"strings"
"time"
"github.com/cairnobs/cairnobs/internal/mapgraph"
"github.com/cairnobs/cairnobs/internal/units"
)
// Root-cause grouping: when something fails and
// something it depends on has failed too, only the deeper failure becomes
// an incident; the others are listed under it as affected.
//
// What depends on what comes from two places (docs/maps.md, Learned
// edges): topology.yaml's needs, and the service map's edges that are
// declared (topology.yaml, Kubernetes, confirmed on the page) or learned
// and steady. A learned edge only seen so far, or fading, never groups,
// and a rejected one is gone from the map. An incident grouped by a
// learned edge says which, so a wrong guess is out in the open, and the
// safety rules stay: the group takes its worst severity, and an alert
// already sent stays on its own.
// GroupedBy is a learned dependency that grouped an incident.
type GroupedBy struct {
From string `json:"from"` // service ids on the map
To string `json:"to"`
FromName string `json:"from_name"`
ToName string `json:"to_name"`
Port int `json:"port,omitempty"`
Since time.Time `json:"since"` // first seen
}
func (g GroupedBy) String() string {
p := ""
if g.Port != 0 {
p = fmt.Sprintf(" :%d", g.Port)
}
return fmt.Sprintf("%s has talked to %s%s since %s", g.FromName, g.ToName, p, g.Since.Format("Jan 2"))
}
// depEdge is one direct need; why is set when it's a learned edge.
type depEdge struct {
to string
why *GroupedBy
}
// deps is what grouping works from: nodes are topology.yaml's services,
// the map's services and hosts.
type deps struct {
topo *Topology
needs map[string][]depEdge
names map[string]string
parts map[string]string // mapgraph.PartKey → the map service it's part of
svc map[string]bool // map services, for a link to the service map
hosts map[string]bool
}
func (d *deps) add(from, to string, why *GroupedBy) {
if from == "" || to == "" || from == to {
return
}
for i, e := range d.needs[from] {
if e.to == to {
if e.why != nil && why == nil { // declared beats learned
d.needs[from][i].why = nil
}
return
}
}
d.needs[from] = append(d.needs[from], depEdge{to, why})
}
// dependencies gathers topology.yaml's and the map's dependencies; nil
// when there are neither.
func (srv *Server) dependencies(now time.Time) *deps {
t := srv.currentTopology()
d := &deps{topo: t, needs: map[string][]depEdge{}, names: map[string]string{}, parts: map[string]string{},
svc: map[string]bool{}, hosts: map[string]bool{}}
if t != nil {
for _, s := range t.Services {
d.names[s.ID] = s.Name
for _, n := range s.Needs {
d.add(s.ID, n, nil)
}
}
}
g := srv.mapGraph(now)
for _, h := range g.Hosts {
d.hosts[h.Name] = true
}
for _, s := range g.Services {
d.svc[s.ID] = true
if d.names[s.ID] == "" {
d.names[s.ID] = s.Name
}
var on []string
for _, p := range s.Parts {
d.parts[mapgraph.PartKey(p.Host, p.Owner)] = s.ID
on = appendNew(on, p.Host)
}
if len(on) == 1 && s.Source != "kubernetes" { // all of it on one host: that host going takes it down
d.add(s.ID, on[0], nil)
}
}
for _, e := range srv.mapEdges(g, now) {
switch e.Stage {
case stageDeclared:
d.add(e.From, e.To, nil)
case stageSteady:
why := &GroupedBy{From: e.From, To: e.To, FromName: d.names[e.From], ToName: d.names[e.To], Port: e.Port}
if e.First != nil {
why.Since = *e.First
}
d.add(e.From, e.To, why)
}
}
// A host that's a VM needs that VM, and the VM its node (vmlink.go):
// keyed as their conditions are.
byHost, _ := srv.vmLinks()
for h, v := range byHost {
d.add(h, v.key, nil)
d.names[v.key] = "VM " + where(v.Name, v.Namespace) + " (" + v.place() + ")"
if v.Node != "" {
n := "node/" + v.Source + "/" + v.Node
d.add(v.key, n, nil)
d.names[n] = "node " + v.Node + " (" + v.Source + ")"
}
}
// A libvirt VM needs the host it runs on: that host silent explains it,
// and through it the guest's own agent going quiet.
for _, h := range srv.store.HostVMs() {
for _, r := range h.vms.List {
k := "vm/" + h.name + "//" + r.Name
d.add(k, h.name, nil)
d.names[k] = "VM " + r.Name + " (" + h.name + ")"
}
}
if t == nil && len(d.needs) == 0 {
return nil
}
return d
}
// nodeOf says which node a condition belongs to: a service, a host, or ""
// when nothing covers it (it then stands alone).
func (d *deps) nodeOf(c Condition) string {
n := ""
if d.topo != nil {
n = nodeOf(d.topo, c)
}
parts := strings.SplitN(c.Key, "/", 3)
switch parts[0] {
case "silent":
return c.Host
case "vm", "node": // a cluster's VM or node, as dependencies keys them
return c.Key
case "unit", "container":
if len(parts) < 3 {
return n
}
host := parts[1]
if n != "" && n != host {
return n // topology.yaml placed it
}
if s := d.parts[mapgraph.PartKey(host, parts[0]+":"+parts[2])]; s != "" {
return s
}
return host
case "workload": // workload/<cluster>/<ns>/<controller>: its service on the map
if len(parts) == 3 {
if s := d.parts[mapgraph.PartKey(parts[1], "k8s:"+parts[2])]; s != "" {
return s
}
}
}
return n
}
// mapHref is where a node is on the maps, or "".
func (d *deps) mapHref(node string) string {
switch {
case d.svc[node]:
return "/map/services?focus=svc:" + node
case d.hosts[node]:
return "/map/network?focus=host:" + node
}
return ""
}
// path is a shortest way from one node to another through what each
// needs, as the edges taken; nil when there's none.
func (d *deps) path(from, to string) []depEdge {
type step struct {
prev string
e depEdge
}
seen := map[string]step{from: {}}
queue := []string{from}
for len(queue) > 0 {
n := queue[0]
queue = queue[1:]
if n == to {
var out []depEdge
for n != from {
s := seen[n]
out = append(out, s.e)
n = s.prev
}
slices.Reverse(out)
return out
}
for _, e := range d.needs[n] {
if _, ok := seen[e.to]; !ok {
seen[e.to] = step{n, e}
queue = append(queue, e.to)
}
}
}
return nil
}
// reach is everything a node depends on, all the way down.
func (d *deps) reach(n string) map[string]bool {
seen := map[string]bool{}
todo := []string{n}
for len(todo) > 0 {
x := todo[len(todo)-1]
todo = todo[:len(todo)-1]
for _, e := range d.needs[x] {
if !seen[e.to] {
seen[e.to] = true
todo = append(todo, e.to)
}
}
}
return seen
}
// nodeOf says which topology.yaml node a condition belongs to: a service,
// a host, or "" when the file doesn't cover it.
func nodeOf(t *Topology, c Condition) string {
parts := strings.SplitN(c.Key, "/", 3)
kind := parts[0]
switch kind {
case "silent":
return c.Host // a host that's gone explains everything on it
case "disk", "memory", "containers-unreadable":
return "" // trouble on a host, but not why its services fail: stands alone
case "unit", "container":
if len(parts) < 3 {
return ""
}
host, name := parts[1], parts[2]
for _, s := range t.Services {
pats := s.Containers
if kind == "unit" {
pats = nil
for _, u := range s.Units {
h, n, _ := strings.Cut(u, "/")
pats = append(pats, h+"/"+units.Name(n))
}
}
for _, p := range pats {
if hostMatch(p, host, name) {
return s.ID
}
}
}
return host // not in any service: at least it's on that host
case "check", "cert":
id := strings.TrimPrefix(c.Key, kind+"/")
for _, s := range t.Services {
if slices.Contains(s.Checks, id) {
return s.ID
}
}
case "drift": // drift/<container>: the service that runs it
name := strings.TrimPrefix(c.Key, "drift/")
for _, s := range t.Services {
for _, p := range s.Containers {
if _, n, _ := strings.Cut(p, "/"); func() bool { ok, _ := path.Match(n, name); return ok }() {
return s.ID
}
}
}
return ""
case "dl": // data layer: dl/<service>/...; the service map ids match (patroni, etcd, haproxy, nats, garage)
if len(parts) > 1 {
for _, s := range t.Services {
if s.ID == parts[1] {
return s.ID
}
}
}
return c.Host
case "mail": // the monitor account's view of mail: the service map's mail node
for _, s := range t.Services {
if s.ID == "mail" {
return s.ID
}
}
return ""
case "akamai":
if len(parts) > 1 && parts[1] == "server" { // GTM can't reach a host: that's the host's trouble
return c.Host
}
case "cloudflare":
if len(parts) > 1 && parts[1] == "origin" { // Cloudflare can't reach a host: that's the host's trouble
return c.Host
}
for _, s := range t.Services {
if s.Cloudflare {
return s.ID
}
}
}
return ""
}
// groupConditions folds conditions by cause, then folds what's alike into
// batches (batches.go).
func (srv *Server) groupConditions(conds []Condition, firing func(key string) bool) []Condition {
return foldBatches(srv.groupByCause(conds, firing), firing)
}
// groupByCause folds conditions caused by a deeper failure into the
// condition for that failure. firing says whether a key already has an
// incident that people were told about: such a condition stays on its
// own, so it neither vanishes nor reads as "resolved" because its cause
// was spotted later.
func (srv *Server) groupByCause(conds []Condition, firing func(key string) bool) []Condition {
if len(conds) == 0 {
return conds
}
d := srv.dependencies(time.Now())
if d == nil {
return conds
}
node := make([]string, len(conds))
failing := map[string]bool{}
for i, c := range conds {
node[i] = d.nodeOf(c)
if node[i] != "" {
failing[node[i]] = true
}
if conds[i].Map == "" {
conds[i].Map = d.mapHref(node[i])
}
}
if len(conds) < 2 {
return conds
}
// What each failing node depends on, worked out only for those.
below := map[string]map[string]bool{}
for n := range failing {
below[n] = d.reach(n)
}
// A failing node is a root when nothing it needs is failing.
isRoot := func(n string) bool {
for m := range below[n] {
if failing[m] {
return false
}
}
return true
}
// The condition that speaks for each root node: the most severe, then
// the first by key, so the choice is stable from one evaluation to the next.
order := make([]int, len(conds))
for i := range order {
order[i] = i
}
sort.SliceStable(order, func(a, b int) bool {
ca, cb := conds[order[a]], conds[order[b]]
if ca.Severity != cb.Severity {
return ca.Severity == "critical"
}
return ca.Key < cb.Key
})
speaker := map[string]int{}
for _, i := range order {
if n := node[i]; n != "" && isRoot(n) {
if _, ok := speaker[n]; !ok {
speaker[n] = i
}
}
}
type group struct {
affected []string // node names, in order of appearance
held int
severity string
why []GroupedBy
}
groups := map[int]*group{}
keep := make([]bool, len(conds))
for _, i := range order {
c, n := conds[i], node[i]
keep[i] = true
if n == "" || isRoot(n) || firing(c.Key) {
continue
}
// Hand it to the first root (in speaker order) that it depends on.
root := -1
for _, j := range order {
if m := node[j]; m != "" && speaker[m] == j && below[n][m] {
root = j
break
}
}
if root < 0 {
continue
}
keep[i] = false
g := groups[root]
if g == nil {
g = &group{severity: conds[root].Severity}
groups[root] = g
}
g.held++
label := cmp.Or(d.names[n], n)
if !slices.Contains(g.affected, label) {
g.affected = append(g.affected, label)
}
if c.Severity == "critical" {
g.severity = "critical"
}
for _, e := range d.path(n, node[root]) {
if e.why != nil && !slices.Contains(g.why, *e.why) {
g.why = append(g.why, *e.why)
}
}
}
out := make([]Condition, 0, len(conds))
for i, c := range conds {
if !keep[i] {
continue
}
if g := groups[i]; g != nil {
c.Severity = g.severity
c.Affected = strings.Join(g.affected, ", ")
c.Grouped = g.why
d := strings.TrimSpace(c.Detail)
if d != "" && !strings.HasSuffix(d, ".") {
d += "."
}
d += fmt.Sprintf(" Also affected: %s (%s held back).", c.Affected, plural(g.held, "alert", "alerts"))
if len(g.why) > 0 {
var ws []string
for _, w := range g.why[:min(len(g.why), 3)] {
ws = append(ws, w.String())
}
more := ""
if len(g.why) > 3 {
more = fmt.Sprintf(" (and %d more)", len(g.why)-3)
}
d += " Grouped because " + strings.Join(ws, "; ") + more + "."
}
c.Detail = strings.TrimSpace(d)
}
out = append(out, c)
}
return out
}
// needsClosure is every topology.yaml node each node depends on, all the
// way down, for the hand-drawn service map.
func needsClosure(t *Topology) map[string]map[string]bool {
d := &deps{needs: map[string][]depEdge{}}
for _, s := range t.Services {
for _, n := range s.Needs {
d.add(s.ID, n, nil)
}
}
out := map[string]map[string]bool{}
for _, s := range t.Services {
out[s.ID] = d.reach(s.ID)
}
return out
}