Files
cairnobs/internal/server/alerts.go
T
jcoffey-dev 30a66b5dd5 alerts: notices give "Since" in the reports' timezone
The text every channel gets said "Since: 2026-11-15 02:57 UTC", which
people outside UTC had to convert. It now uses reports.timezone (the
server's own when that's left out), as the Reports page and the digest
do: "Since: 2026-11-14 19:57 MST". Machine timestamps (PagerDuty's,
Discord's embed) stay UTC, as those APIs expect.
2026-11-14 20:12:15 -07:00

704 lines
21 KiB
Go

package server
// The alert engine. Every evaluation, the conditions from view() are matched
// to incidents by key:
//
// - a new condition opens an incident, but nobody is told until it has
// lasted alerts.for (2 minutes), so a blip doesn't page anyone;
// - once it fires, every channel whose minimum it reaches is told;
// - a critical incident nobody has acknowledged is repeated every
// alerts.repeat (15 minutes); one that gets worse is sent again at once;
// - when the condition goes away the incident resolves, and the channels
// that heard about it hear that too. One that never fired just goes.
//
// Notices are sent apart from evaluation, with retries, so a slow channel
// never holds it up. With a database, incidents are kept there, so a
// restart (or another copy taking over as evaluator, lead.go) carries on
// with them, and so are the notices: each goes into the outbox in the same
// transaction as its incident's save, and the evaluator sends from there
// (outbox.go). An evaluator whose lease has passed (the save is refused)
// queues nothing; a notice it did queue is sent by whoever takes over.
// Without a database, notices go through a queue in memory.
import (
"context"
"encoding/json"
"fmt"
"log"
"net/http"
"reflect"
"slices"
"sort"
"sync"
"time"
"github.com/cairnobs/cairnobs/internal/notify"
"github.com/cairnobs/cairnobs/internal/state"
)
// AlertsConfig is the alerts section of obs-beacond.yaml.
type AlertsConfig struct {
For time.Duration `yaml:"for"` // how long a problem lasts before anyone is told (2m)
Repeat time.Duration `yaml:"repeat"` // how often an unacknowledged critical is repeated (15m)
PublicURL string `yaml:"public_url"` // Cairn OBS's address, for links in notices
Channels []notify.Config `yaml:"channels"`
}
// Incident is one problem, from first seen to resolved.
type Incident struct {
ID string `json:"id"`
Key string `json:"key"`
Severity string `json:"severity"`
Title string `json:"title"`
Detail string `json:"detail"`
Host string `json:"host"`
Href string `json:"href"`
Opened time.Time `json:"opened"`
Fired *time.Time `json:"fired"` // when people were first told; nil while holding back
Notified *time.Time `json:"notified"` // the last notice
Resolved *time.Time `json:"resolved"`
Ack *Ack `json:"ack"`
Affected string `json:"affected,omitempty"` // what else it takes down, from root-cause grouping
Grouped []GroupedBy `json:"grouped,omitempty"` // the learned dependencies that grouped it
Map string `json:"map,omitempty"` // where it is on the maps
}
// same says whether nothing worth saving changed.
func (i Incident) same(o Incident) bool {
g1, g2 := i.Grouped, o.Grouped
i.Grouped, o.Grouped = nil, nil
return reflect.DeepEqual(i, o) && slices.Equal(g1, g2)
}
// grouping is what's kept of an incident's root-cause grouping.
type grouping struct {
Affected string `json:"affected,omitempty"`
Grouped []GroupedBy `json:"grouped,omitempty"`
Map string `json:"map,omitempty"`
}
// Ack records who acknowledged an incident.
type Ack struct {
By string `json:"by"`
At time.Time `json:"at"`
}
type job struct {
n notify.Notice
chans []notify.Channel
inc string // the incident it's about; saved with it into the outbox
}
type alerter struct {
cfg AlertsConfig
chMu sync.RWMutex
channels []notify.Channel // replaced whole when the UI changes them; read with chans()
retry []time.Duration // waits between attempts
db *state.DB // nil: incidents live in memory only
loc *time.Location // notices give times here: the reports' timezone (nil: UTC)
mu sync.Mutex
open map[string]*Incident // by condition key
history []Incident // resolved, newest first
seq int
queue chan job
pend []job // notices waiting for their incident's save (under mu)
fence *state.Fence // the evaluator lease, when copies share the database (under mu)
fenced bool // a save was refused in this round (under mu)
onFenced func() // told when a save is refused: this copy lost the lease
changed bool // something was saved in this round (under mu)
wake chan struct{} // the outbox has something new
}
const historyKeep = 200
func newAlerter(cfg AlertsConfig) (*alerter, error) {
if cfg.For == 0 {
cfg.For = 2 * time.Minute
}
if cfg.Repeat == 0 {
cfg.Repeat = 15 * time.Minute
}
a := &alerter{cfg: cfg, open: map[string]*Incident{}, queue: make(chan job, 256), wake: make(chan struct{}, 1), retry: []time.Duration{5 * time.Second, 30 * time.Second}}
chans, err := buildChannels(cfg.Channels)
if err != nil {
return nil, err
}
a.channels = chans
return a, nil
}
// buildChannels makes the channels from their settings, reading their
// secrets now, and checks them as a set.
func buildChannels(cfgs []notify.Config) ([]notify.Channel, error) {
var out []notify.Channel
ids := map[string]bool{}
for _, c := range cfgs {
ch, err := notify.New(c)
if err != nil {
return nil, err
}
if ids[ch.ID()] {
return nil, fmt.Errorf("two alert channels called %q", ch.ID())
}
ids[ch.ID()] = true
out = append(out, ch)
}
// If mail is what's down, an email about it never arrives: critical
// alerts must also reach at least one channel that isn't email.
mailOnly := false
for _, ch := range out {
if notify.Rank(ch.Min()) <= notify.Rank("critical") {
if ch.Type() != "email" {
return out, nil
}
mailOnly = true
}
}
if mailOnly {
return nil, fmt.Errorf("critical alerts would only go out by email; add a channel that isn't email (ntfy, Discord, webhook, PagerDuty) taking critical")
}
return out, nil
}
// chans are the channels in force. The slice is never changed in place.
func (a *alerter) chans() []notify.Channel {
a.chMu.RLock()
defer a.chMu.RUnlock()
return a.channels
}
func (a *alerter) setChannels(chans []notify.Channel) {
a.chMu.Lock()
a.channels = chans
a.chMu.Unlock()
}
func (a *alerter) zone() *time.Location {
if a.loc == nil {
return time.UTC
}
return a.loc
}
func (a *alerter) notice(inc *Incident, kind string) notify.Notice {
n := notify.Notice{Kind: kind, ID: inc.ID, Severity: inc.Severity, Title: inc.Title, Detail: inc.Detail, Host: inc.Host, Since: inc.Opened.In(a.zone())}
if a.cfg.PublicURL != "" && inc.Href != "" {
n.URL = a.cfg.PublicURL + inc.Href
}
return n
}
// enqueue sends to every channel whose minimum the severity reaches.
// Called with a.mu held.
func (a *alerter) enqueue(inc *Incident, kind string, now time.Time) {
t := now
inc.Notified = &t
var to []notify.Channel
for _, ch := range a.chans() {
if notify.Rank(inc.Severity) >= notify.Rank(ch.Min()) {
to = append(to, ch)
}
}
if len(to) == 0 {
return
}
a.pend = append(a.pend, job{n: a.notice(inc, kind), chans: to, inc: inc.ID})
}
// tellFollowers tells the channels that follow incidents (PagerDuty) about
// one they were told of; the others hear only of new and resolved
// problems. Called with a.mu held.
func (a *alerter) tellFollowers(inc *Incident, kind string) {
if inc.Fired == nil {
return // nobody was told of it
}
var to []notify.Channel
for _, ch := range a.chans() {
if f, ok := ch.(notify.Follower); ok && f.Follows() && notify.Rank(inc.Severity) >= notify.Rank(ch.Min()) {
to = append(to, ch)
}
}
if len(to) == 0 {
return
}
a.pend = append(a.pend, job{n: a.notice(inc, kind), chans: to})
}
// flush queues the notices of a round not already saved with their
// incident, unless a save in it was refused; the caller holds a.mu.
func (a *alerter) flush() {
pend := a.pend
a.pend = nil
if a.fenced {
if len(pend) > 0 {
log.Printf("alerts: not sending %d notices: another copy is the evaluator now", len(pend))
}
return
}
if a.db != nil {
if len(pend) == 0 {
return
}
err := a.db.QueueNotices(outboxItems(pend, time.Now()), a.fence)
a.checkFence(err)
switch {
case err == nil:
a.poke()
return
case err == state.ErrFenced:
return
}
// The database is away: send them from memory rather than not at all.
log.Printf("alerts: can't queue %d notices in the database, sending them from memory: %v", len(pend), err)
}
for _, j := range pend {
select {
case a.queue <- j:
default:
log.Printf("alerts: queue full, dropped %s %s", j.n.Kind, j.n.ID)
}
}
}
// tellEveryone sends a notice to every channel, whatever its minimum: for
// things everyone must hear about, like the emergency door being used.
func (a *alerter) tellEveryone(n notify.Notice) {
chans := a.chans()
if len(chans) == 0 {
return
}
n.URL = a.cfg.PublicURL
select {
case a.queue <- job{n: n, chans: chans}:
default:
log.Printf("alerts: queue full, dropped %q", n.Title)
}
}
func t0(p *time.Time) time.Time {
if p == nil {
return time.Time{}
}
return *p
}
func tp(t time.Time) *time.Time {
if t.IsZero() {
return nil
}
return &t
}
// save writes an incident through to the database. Called with a.mu held.
func (a *alerter) save(inc *Incident) {
if a.db == nil {
return
}
si := state.Incident{ID: inc.ID, Key: inc.Key, Severity: inc.Severity, Title: inc.Title, Detail: inc.Detail, Host: inc.Host, Href: inc.Href,
Opened: inc.Opened, Fired: t0(inc.Fired), Notified: t0(inc.Notified), Resolved: t0(inc.Resolved)}
if inc.Ack != nil {
si.AckBy, si.AckAt = inc.Ack.By, inc.Ack.At
}
if inc.Affected != "" || len(inc.Grouped) > 0 || inc.Map != "" {
b, _ := json.Marshal(grouping{inc.Affected, inc.Grouped, inc.Map})
si.Grouping = string(b)
}
a.changed = true
// The notices about it go into the outbox with it.
var mine, rest []job
for _, j := range a.pend {
if j.inc == inc.ID {
mine = append(mine, j)
} else {
rest = append(rest, j)
}
}
err := a.db.PutIncidentNotices(si, a.fence, outboxItems(mine, time.Now()))
a.checkFence(err)
if err != nil {
if err != state.ErrFenced {
log.Printf("alerts: can't store %s: %v", inc.ID, err)
}
return // its notices stay pending: flush drops them if fenced, else queues them
}
a.pend = rest
if len(mine) > 0 {
a.poke()
}
}
// checkFence notes a refused save: this copy isn't the evaluator any more.
// The caller holds a.mu.
func (a *alerter) checkFence(err error) {
if err != state.ErrFenced || a.fenced {
return
}
a.fenced = true
a.fence = nil
log.Printf("alerts: another copy has taken over as evaluator")
if a.onFenced != nil {
a.onFenced()
}
}
// lead makes this copy the evaluator under f (nil: it isn't one), starting
// from the incidents as stored.
func (a *alerter) lead(f *state.Fence) {
if f != nil {
a.load()
}
a.mu.Lock()
defer a.mu.Unlock()
a.fence, a.fenced, a.pend = f, false, nil
}
func fromState(si state.Incident) *Incident {
inc := &Incident{ID: si.ID, Key: si.Key, Severity: si.Severity, Title: si.Title, Detail: si.Detail, Host: si.Host, Href: si.Href,
Opened: si.Opened, Fired: tp(si.Fired), Notified: tp(si.Notified), Resolved: tp(si.Resolved)}
if si.AckBy != "" {
inc.Ack = &Ack{By: si.AckBy, At: si.AckAt}
}
var g grouping
if si.Grouping != "" && json.Unmarshal([]byte(si.Grouping), &g) == nil {
inc.Affected, inc.Grouped, inc.Map = g.Affected, g.Grouped, g.Map
}
return inc
}
// load reads the incidents as stored, replacing what's in memory: at
// start, on taking over as evaluator, and on copies that aren't the
// evaluator whenever it says they changed. After a restart that means an
// upgrade neither forgets an alert nor sends it again.
func (a *alerter) load() {
if a.db == nil {
return
}
open, recent, err := a.db.Incidents(historyKeep)
if err != nil {
log.Printf("alerts: can't load incidents: %v", err)
return
}
a.mu.Lock()
defer a.mu.Unlock()
first := len(a.open) == 0 && len(a.history) == 0
a.open, a.history = map[string]*Incident{}, nil
for _, si := range open {
a.open[si.Key] = fromState(si)
}
for _, si := range recent {
a.history = append(a.history, *fromState(si))
}
fmt.Sscan(a.db.Meta("incident_seq"), &a.seq)
if first && len(open) > 0 {
log.Printf("alerts: %d open incidents brought back from the database", len(open))
}
}
// evaluate matches this round's conditions to incidents.
func (a *alerter) evaluate(now time.Time, conds []Condition) { a.evaluateHeld(now, conds, nil) }
// evaluateHeld is evaluate with some hosts in a maintenance window: their
// problems don't open or fire anything, and incidents already open for them
// stay exactly as they are (no repeats, and no "resolved" while the host is
// simply rebooting). When the window closes they're judged normally again.
func (a *alerter) evaluateHeld(now time.Time, conds []Condition, held map[string]bool) bool {
a.mu.Lock()
defer a.mu.Unlock()
a.changed = false
defer a.flush()
seen := map[string]bool{}
for _, c := range conds {
if c.Host != "" && held[c.Host] {
continue
}
seen[c.Key] = true
inc := a.open[c.Key]
if inc == nil {
a.seq++
if a.db != nil {
a.checkFence(a.db.SetMetaAs("incident_seq", fmt.Sprint(a.seq), a.fence))
}
inc = &Incident{ID: fmt.Sprintf("B-%04d", a.seq), Key: c.Key, Opened: now}
a.open[c.Key] = inc
}
before := *inc
worse := inc.Fired != nil && notify.Rank(c.Severity) > notify.Rank(inc.Severity)
inc.Severity, inc.Title, inc.Detail, inc.Host, inc.Href, inc.Affected = c.Severity, c.Title, c.Detail, c.Host, c.Href, c.Affected
inc.Grouped, inc.Map = c.Grouped, c.Map
switch {
case inc.Fired == nil && now.Sub(inc.Opened) >= a.cfg.For:
t := now
inc.Fired = &t
a.enqueue(inc, "firing", now)
case worse:
if inc.Ack != nil && a.db != nil && !a.fenced {
_ = a.db.ClearAck(inc.ID)
}
inc.Ack = nil // it changed; someone needs to look again
a.enqueue(inc, "escalated", now)
case inc.Fired != nil && inc.Severity == "critical" && inc.Ack == nil && now.Sub(*inc.Notified) >= a.cfg.Repeat:
a.enqueue(inc, "still", now)
}
if !inc.same(before) || before.ID == "" || inc.Opened.Equal(now) {
a.save(inc)
}
}
for key, inc := range a.open {
if seen[key] || (inc.Host != "" && held[inc.Host]) {
continue
}
delete(a.open, key)
if inc.Fired == nil {
if a.db != nil && !a.fenced { // a blip shorter than alerts.for: nobody was told, nothing to keep
_ = a.db.DeleteIncident(inc.ID)
a.changed = true
}
continue
}
t := now
inc.Resolved = &t
a.enqueue(inc, "resolved", now)
a.save(inc)
a.history = append([]Incident{*inc}, a.history...)
if len(a.history) > historyKeep {
a.history = a.history[:historyKeep]
}
}
return a.changed
}
// acknowledge stops the repeats for an open incident. With a database the
// acknowledgement is written there on its own, so any copy may take it;
// tell says whether this copy also tells the channels that follow
// incidents (it's the evaluator, or the only copy).
func (a *alerter) acknowledge(id, by string, now time.Time, tell bool) (Incident, bool) {
a.mu.Lock()
defer a.mu.Unlock()
for _, inc := range a.open {
if inc.ID == id {
if a.db != nil {
if ok, err := a.db.AckIncident(id, by, now); err != nil || !ok {
return Incident{}, false
}
}
inc.Ack = &Ack{By: by, At: now}
if tell {
a.tellFollowers(inc, "acknowledged")
a.flush()
}
return *inc, true
}
}
return Incident{}, false
}
// acknowledged takes in an acknowledgement another copy made, as stored,
// and, when tell, tells the channels that follow incidents.
func (a *alerter) acknowledged(id string, tell bool) {
si, err := a.db.Incident(id)
if err != nil || si.AckBy == "" {
return
}
a.mu.Lock()
defer a.mu.Unlock()
for _, inc := range a.open {
if inc.ID == id && inc.Ack == nil {
inc.Ack = &Ack{By: si.AckBy, At: si.AckAt}
if tell {
a.tellFollowers(inc, "acknowledged")
a.flush()
}
}
}
}
// snapshot lists open incidents (critical first, then oldest) and recent history.
// firing says whether the condition with this key already has an incident
// people were told about.
func (a *alerter) firing(key string) bool {
a.mu.Lock()
defer a.mu.Unlock()
inc := a.open[key]
return inc != nil && inc.Fired != nil
}
func (a *alerter) snapshot() (open, recent []Incident) {
a.mu.Lock()
defer a.mu.Unlock()
open = []Incident{}
for _, inc := range a.open {
open = append(open, *inc)
}
sort.Slice(open, func(i, j int) bool {
if ri, rj := notify.Rank(open[i].Severity), notify.Rank(open[j].Severity); ri != rj {
return ri > rj
}
return open[i].Opened.Before(open[j].Opened)
})
n := min(len(a.history), 50)
recent = append([]Incident{}, a.history[:n]...)
return open, recent
}
// run delivers queued notices until ctx ends.
func (a *alerter) run(ctx context.Context) {
for {
select {
case <-ctx.Done():
return
case j := <-a.queue:
for _, ch := range j.chans {
a.deliver(ctx, ch, j.n)
}
}
}
}
func (a *alerter) deliver(ctx context.Context, ch notify.Channel, n notify.Notice) {
for attempt := 0; ; attempt++ {
sctx, cancel := context.WithTimeout(ctx, 15*time.Second)
err := ch.Send(sctx, n)
cancel()
if err == nil {
log.Printf("alerts: sent %s %s (%s) to %s", n.Kind, n.ID, n.Severity, ch.ID())
return
}
if attempt >= len(a.retry) {
log.Printf("alerts: gave up on %s for %s %s: %v", ch.ID(), n.Kind, n.ID, err)
return
}
log.Printf("alerts: %v; trying again in %s", err, a.retry[attempt])
select {
case <-ctx.Done():
return
case <-time.After(a.retry[attempt]):
}
}
}
// test sends a test notice right away to every channel, or to the one
// called only, and reports each result.
func (a *alerter) test(ctx context.Context, only string) []map[string]string {
out := []map[string]string{}
for _, ch := range a.chans() {
if only != "" && ch.ID() != only {
continue
}
sctx, cancel := context.WithTimeout(ctx, 15*time.Second)
err := ch.Send(sctx, notify.Notice{Kind: "test", Severity: "warning", Title: "Test", Since: time.Now().In(a.zone()), URL: a.cfg.PublicURL})
cancel()
res := map[string]string{"id": ch.ID(), "type": ch.Type(), "result": "sent"}
if err != nil {
res["result"] = err.Error()
}
log.Printf("alerts: test to %s: %s", ch.ID(), res["result"])
out = append(out, res)
}
return out
}
// --- HTTP ---
// evaluate runs one round of the alerts and says whether any incident
// changed.
func (srv *Server) evaluate(now time.Time) bool {
// Right after a start nothing has reported yet, and incidents brought
// back from the database would look fixed: wait until every agent has
// had time to report before judging anything.
if now.Sub(srv.started) < srv.cfg.StaleAfter {
return false
}
_, conds := srv.view(now)
conds = srv.groupConditions(conds, srv.alerts.firing)
return srv.alerts.evaluateHeld(now, conds, srv.heldHosts(now))
}
// IncidentView is an open incident as the Alerts page shows it: with the
// last log lines when it's about a container that has them. Logs stay in
// Cairn OBS: notices carry the link, not the lines.
type IncidentView struct {
Incident
Logs []string `json:"logs,omitempty"`
}
func (srv *Server) handleIncidents(w http.ResponseWriter, r *http.Request) {
snapOpen, recent := srv.alerts.snapshot()
hosts, _ := srv.view(time.Now())
logs := map[string][]string{}
for _, h := range hosts {
if !srv.seesAll(r) {
break // log lines are for admins and operators
}
for _, row := range h.rows {
if len(row.Logs) > 0 {
logs["container/"+h.Name+"/"+row.Name] = row.Logs
}
}
}
open := make([]IncidentView, 0, len(snapOpen))
for _, inc := range snapOpen {
open = append(open, IncidentView{Incident: inc, Logs: logs[inc.Key]})
}
chans := []map[string]string{}
for _, ch := range srv.alerts.chans() {
chans = append(chans, map[string]string{"id": ch.ID(), "type": ch.Type(), "min": ch.Min()})
}
writeJSON(w, http.StatusOK, map[string]any{
"open": open, "recent": recent, "channels": chans,
"for_s": int(srv.alerts.cfg.For / time.Second), "repeat_s": int(srv.alerts.cfg.Repeat / time.Second), "stored": srv.db != nil,
"maintenance": srv.maintenanceView(time.Now()), "maintenance_source": srv.maintenanceSource(), "channels_source": srv.channelsSource(),
})
}
func (srv *Server) handleAck(w http.ResponseWriter, r *http.Request) {
_, s, _ := srv.sessions.get(r)
lead := srv.leading()
inc, ok := srv.alerts.acknowledge(r.PathValue("id"), s.User, time.Now(), lead)
if !ok {
writeJSON(w, http.StatusNotFound, map[string]string{"error": "no open incident with that id"})
return
}
if !lead {
srv.publish(noticeAck, inc.ID) // the evaluator tells the channels that follow incidents
}
srv.audit(s.User, "acknowledged", inc.ID+": "+inc.Title)
writeJSON(w, http.StatusOK, inc)
}
func (srv *Server) handleAlertTest(w http.ResponseWriter, r *http.Request) {
_, s, _ := srv.sessions.get(r)
only := r.URL.Query().Get("channel")
srv.audit(s.User, "sent a test alert", only)
writeJSON(w, http.StatusOK, map[string]any{"channels": srv.alerts.test(r.Context(), only)})
}
// heldHosts are the hosts in a maintenance window right now.
func (srv *Server) heldHosts(now time.Time) map[string]bool {
held := map[string]bool{}
ws := srv.windows()
for i := range ws {
w := &ws[i]
if w.Active(now) {
for _, h := range w.Hosts {
held[h] = true
}
}
}
return held
}
// maintenanceView is what the Alerts page shows about the windows.
func (srv *Server) maintenanceView(now time.Time) []map[string]any {
out := []map[string]any{}
ws := srv.windows()
for i := range ws {
w := &ws[i]
out = append(out, map[string]any{"name": w.Name, "hosts": w.Hosts, "daily": w.Daily, "timezone": w.Timezone,
"duration_s": int(w.Duration / time.Second), "active": w.Active(now), "next": w.Next(now)})
}
return out
}