The text every channel gets said "Since: 2026-11-15 02:57 UTC", which people outside UTC had to convert. It now uses reports.timezone (the server's own when that's left out), as the Reports page and the digest do: "Since: 2026-11-14 19:57 MST". Machine timestamps (PagerDuty's, Discord's embed) stay UTC, as those APIs expect.
704 lines
21 KiB
Go
704 lines
21 KiB
Go
package server
|
|
|
|
// The alert engine. Every evaluation, the conditions from view() are matched
|
|
// to incidents by key:
|
|
//
|
|
// - a new condition opens an incident, but nobody is told until it has
|
|
// lasted alerts.for (2 minutes), so a blip doesn't page anyone;
|
|
// - once it fires, every channel whose minimum it reaches is told;
|
|
// - a critical incident nobody has acknowledged is repeated every
|
|
// alerts.repeat (15 minutes); one that gets worse is sent again at once;
|
|
// - when the condition goes away the incident resolves, and the channels
|
|
// that heard about it hear that too. One that never fired just goes.
|
|
//
|
|
// Notices are sent apart from evaluation, with retries, so a slow channel
|
|
// never holds it up. With a database, incidents are kept there, so a
|
|
// restart (or another copy taking over as evaluator, lead.go) carries on
|
|
// with them, and so are the notices: each goes into the outbox in the same
|
|
// transaction as its incident's save, and the evaluator sends from there
|
|
// (outbox.go). An evaluator whose lease has passed (the save is refused)
|
|
// queues nothing; a notice it did queue is sent by whoever takes over.
|
|
// Without a database, notices go through a queue in memory.
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"log"
|
|
"net/http"
|
|
"reflect"
|
|
"slices"
|
|
"sort"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/cairnobs/cairnobs/internal/notify"
|
|
"github.com/cairnobs/cairnobs/internal/state"
|
|
)
|
|
|
|
// AlertsConfig is the alerts section of obs-beacond.yaml.
|
|
type AlertsConfig struct {
|
|
For time.Duration `yaml:"for"` // how long a problem lasts before anyone is told (2m)
|
|
Repeat time.Duration `yaml:"repeat"` // how often an unacknowledged critical is repeated (15m)
|
|
PublicURL string `yaml:"public_url"` // Cairn OBS's address, for links in notices
|
|
Channels []notify.Config `yaml:"channels"`
|
|
}
|
|
|
|
// Incident is one problem, from first seen to resolved.
|
|
type Incident struct {
|
|
ID string `json:"id"`
|
|
Key string `json:"key"`
|
|
Severity string `json:"severity"`
|
|
Title string `json:"title"`
|
|
Detail string `json:"detail"`
|
|
Host string `json:"host"`
|
|
Href string `json:"href"`
|
|
Opened time.Time `json:"opened"`
|
|
Fired *time.Time `json:"fired"` // when people were first told; nil while holding back
|
|
Notified *time.Time `json:"notified"` // the last notice
|
|
Resolved *time.Time `json:"resolved"`
|
|
Ack *Ack `json:"ack"`
|
|
Affected string `json:"affected,omitempty"` // what else it takes down, from root-cause grouping
|
|
Grouped []GroupedBy `json:"grouped,omitempty"` // the learned dependencies that grouped it
|
|
Map string `json:"map,omitempty"` // where it is on the maps
|
|
}
|
|
|
|
// same says whether nothing worth saving changed.
|
|
func (i Incident) same(o Incident) bool {
|
|
g1, g2 := i.Grouped, o.Grouped
|
|
i.Grouped, o.Grouped = nil, nil
|
|
return reflect.DeepEqual(i, o) && slices.Equal(g1, g2)
|
|
}
|
|
|
|
// grouping is what's kept of an incident's root-cause grouping.
|
|
type grouping struct {
|
|
Affected string `json:"affected,omitempty"`
|
|
Grouped []GroupedBy `json:"grouped,omitempty"`
|
|
Map string `json:"map,omitempty"`
|
|
}
|
|
|
|
// Ack records who acknowledged an incident.
|
|
type Ack struct {
|
|
By string `json:"by"`
|
|
At time.Time `json:"at"`
|
|
}
|
|
|
|
type job struct {
|
|
n notify.Notice
|
|
chans []notify.Channel
|
|
inc string // the incident it's about; saved with it into the outbox
|
|
}
|
|
|
|
type alerter struct {
|
|
cfg AlertsConfig
|
|
|
|
chMu sync.RWMutex
|
|
channels []notify.Channel // replaced whole when the UI changes them; read with chans()
|
|
retry []time.Duration // waits between attempts
|
|
|
|
db *state.DB // nil: incidents live in memory only
|
|
loc *time.Location // notices give times here: the reports' timezone (nil: UTC)
|
|
|
|
mu sync.Mutex
|
|
open map[string]*Incident // by condition key
|
|
history []Incident // resolved, newest first
|
|
seq int
|
|
queue chan job
|
|
pend []job // notices waiting for their incident's save (under mu)
|
|
fence *state.Fence // the evaluator lease, when copies share the database (under mu)
|
|
fenced bool // a save was refused in this round (under mu)
|
|
onFenced func() // told when a save is refused: this copy lost the lease
|
|
changed bool // something was saved in this round (under mu)
|
|
wake chan struct{} // the outbox has something new
|
|
}
|
|
|
|
const historyKeep = 200
|
|
|
|
func newAlerter(cfg AlertsConfig) (*alerter, error) {
|
|
if cfg.For == 0 {
|
|
cfg.For = 2 * time.Minute
|
|
}
|
|
if cfg.Repeat == 0 {
|
|
cfg.Repeat = 15 * time.Minute
|
|
}
|
|
a := &alerter{cfg: cfg, open: map[string]*Incident{}, queue: make(chan job, 256), wake: make(chan struct{}, 1), retry: []time.Duration{5 * time.Second, 30 * time.Second}}
|
|
chans, err := buildChannels(cfg.Channels)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
a.channels = chans
|
|
return a, nil
|
|
}
|
|
|
|
// buildChannels makes the channels from their settings, reading their
|
|
// secrets now, and checks them as a set.
|
|
func buildChannels(cfgs []notify.Config) ([]notify.Channel, error) {
|
|
var out []notify.Channel
|
|
ids := map[string]bool{}
|
|
for _, c := range cfgs {
|
|
ch, err := notify.New(c)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if ids[ch.ID()] {
|
|
return nil, fmt.Errorf("two alert channels called %q", ch.ID())
|
|
}
|
|
ids[ch.ID()] = true
|
|
out = append(out, ch)
|
|
}
|
|
// If mail is what's down, an email about it never arrives: critical
|
|
// alerts must also reach at least one channel that isn't email.
|
|
mailOnly := false
|
|
for _, ch := range out {
|
|
if notify.Rank(ch.Min()) <= notify.Rank("critical") {
|
|
if ch.Type() != "email" {
|
|
return out, nil
|
|
}
|
|
mailOnly = true
|
|
}
|
|
}
|
|
if mailOnly {
|
|
return nil, fmt.Errorf("critical alerts would only go out by email; add a channel that isn't email (ntfy, Discord, webhook, PagerDuty) taking critical")
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// chans are the channels in force. The slice is never changed in place.
|
|
func (a *alerter) chans() []notify.Channel {
|
|
a.chMu.RLock()
|
|
defer a.chMu.RUnlock()
|
|
return a.channels
|
|
}
|
|
|
|
func (a *alerter) setChannels(chans []notify.Channel) {
|
|
a.chMu.Lock()
|
|
a.channels = chans
|
|
a.chMu.Unlock()
|
|
}
|
|
|
|
func (a *alerter) zone() *time.Location {
|
|
if a.loc == nil {
|
|
return time.UTC
|
|
}
|
|
return a.loc
|
|
}
|
|
|
|
func (a *alerter) notice(inc *Incident, kind string) notify.Notice {
|
|
n := notify.Notice{Kind: kind, ID: inc.ID, Severity: inc.Severity, Title: inc.Title, Detail: inc.Detail, Host: inc.Host, Since: inc.Opened.In(a.zone())}
|
|
if a.cfg.PublicURL != "" && inc.Href != "" {
|
|
n.URL = a.cfg.PublicURL + inc.Href
|
|
}
|
|
return n
|
|
}
|
|
|
|
// enqueue sends to every channel whose minimum the severity reaches.
|
|
// Called with a.mu held.
|
|
func (a *alerter) enqueue(inc *Incident, kind string, now time.Time) {
|
|
t := now
|
|
inc.Notified = &t
|
|
var to []notify.Channel
|
|
for _, ch := range a.chans() {
|
|
if notify.Rank(inc.Severity) >= notify.Rank(ch.Min()) {
|
|
to = append(to, ch)
|
|
}
|
|
}
|
|
if len(to) == 0 {
|
|
return
|
|
}
|
|
a.pend = append(a.pend, job{n: a.notice(inc, kind), chans: to, inc: inc.ID})
|
|
}
|
|
|
|
// tellFollowers tells the channels that follow incidents (PagerDuty) about
|
|
// one they were told of; the others hear only of new and resolved
|
|
// problems. Called with a.mu held.
|
|
func (a *alerter) tellFollowers(inc *Incident, kind string) {
|
|
if inc.Fired == nil {
|
|
return // nobody was told of it
|
|
}
|
|
var to []notify.Channel
|
|
for _, ch := range a.chans() {
|
|
if f, ok := ch.(notify.Follower); ok && f.Follows() && notify.Rank(inc.Severity) >= notify.Rank(ch.Min()) {
|
|
to = append(to, ch)
|
|
}
|
|
}
|
|
if len(to) == 0 {
|
|
return
|
|
}
|
|
a.pend = append(a.pend, job{n: a.notice(inc, kind), chans: to})
|
|
}
|
|
|
|
// flush queues the notices of a round not already saved with their
|
|
// incident, unless a save in it was refused; the caller holds a.mu.
|
|
func (a *alerter) flush() {
|
|
pend := a.pend
|
|
a.pend = nil
|
|
if a.fenced {
|
|
if len(pend) > 0 {
|
|
log.Printf("alerts: not sending %d notices: another copy is the evaluator now", len(pend))
|
|
}
|
|
return
|
|
}
|
|
if a.db != nil {
|
|
if len(pend) == 0 {
|
|
return
|
|
}
|
|
err := a.db.QueueNotices(outboxItems(pend, time.Now()), a.fence)
|
|
a.checkFence(err)
|
|
switch {
|
|
case err == nil:
|
|
a.poke()
|
|
return
|
|
case err == state.ErrFenced:
|
|
return
|
|
}
|
|
// The database is away: send them from memory rather than not at all.
|
|
log.Printf("alerts: can't queue %d notices in the database, sending them from memory: %v", len(pend), err)
|
|
}
|
|
for _, j := range pend {
|
|
select {
|
|
case a.queue <- j:
|
|
default:
|
|
log.Printf("alerts: queue full, dropped %s %s", j.n.Kind, j.n.ID)
|
|
}
|
|
}
|
|
}
|
|
|
|
// tellEveryone sends a notice to every channel, whatever its minimum: for
|
|
// things everyone must hear about, like the emergency door being used.
|
|
func (a *alerter) tellEveryone(n notify.Notice) {
|
|
chans := a.chans()
|
|
if len(chans) == 0 {
|
|
return
|
|
}
|
|
n.URL = a.cfg.PublicURL
|
|
select {
|
|
case a.queue <- job{n: n, chans: chans}:
|
|
default:
|
|
log.Printf("alerts: queue full, dropped %q", n.Title)
|
|
}
|
|
}
|
|
|
|
func t0(p *time.Time) time.Time {
|
|
if p == nil {
|
|
return time.Time{}
|
|
}
|
|
return *p
|
|
}
|
|
|
|
func tp(t time.Time) *time.Time {
|
|
if t.IsZero() {
|
|
return nil
|
|
}
|
|
return &t
|
|
}
|
|
|
|
// save writes an incident through to the database. Called with a.mu held.
|
|
func (a *alerter) save(inc *Incident) {
|
|
if a.db == nil {
|
|
return
|
|
}
|
|
si := state.Incident{ID: inc.ID, Key: inc.Key, Severity: inc.Severity, Title: inc.Title, Detail: inc.Detail, Host: inc.Host, Href: inc.Href,
|
|
Opened: inc.Opened, Fired: t0(inc.Fired), Notified: t0(inc.Notified), Resolved: t0(inc.Resolved)}
|
|
if inc.Ack != nil {
|
|
si.AckBy, si.AckAt = inc.Ack.By, inc.Ack.At
|
|
}
|
|
if inc.Affected != "" || len(inc.Grouped) > 0 || inc.Map != "" {
|
|
b, _ := json.Marshal(grouping{inc.Affected, inc.Grouped, inc.Map})
|
|
si.Grouping = string(b)
|
|
}
|
|
a.changed = true
|
|
// The notices about it go into the outbox with it.
|
|
var mine, rest []job
|
|
for _, j := range a.pend {
|
|
if j.inc == inc.ID {
|
|
mine = append(mine, j)
|
|
} else {
|
|
rest = append(rest, j)
|
|
}
|
|
}
|
|
err := a.db.PutIncidentNotices(si, a.fence, outboxItems(mine, time.Now()))
|
|
a.checkFence(err)
|
|
if err != nil {
|
|
if err != state.ErrFenced {
|
|
log.Printf("alerts: can't store %s: %v", inc.ID, err)
|
|
}
|
|
return // its notices stay pending: flush drops them if fenced, else queues them
|
|
}
|
|
a.pend = rest
|
|
if len(mine) > 0 {
|
|
a.poke()
|
|
}
|
|
}
|
|
|
|
// checkFence notes a refused save: this copy isn't the evaluator any more.
|
|
// The caller holds a.mu.
|
|
func (a *alerter) checkFence(err error) {
|
|
if err != state.ErrFenced || a.fenced {
|
|
return
|
|
}
|
|
a.fenced = true
|
|
a.fence = nil
|
|
log.Printf("alerts: another copy has taken over as evaluator")
|
|
if a.onFenced != nil {
|
|
a.onFenced()
|
|
}
|
|
}
|
|
|
|
// lead makes this copy the evaluator under f (nil: it isn't one), starting
|
|
// from the incidents as stored.
|
|
func (a *alerter) lead(f *state.Fence) {
|
|
if f != nil {
|
|
a.load()
|
|
}
|
|
a.mu.Lock()
|
|
defer a.mu.Unlock()
|
|
a.fence, a.fenced, a.pend = f, false, nil
|
|
}
|
|
|
|
func fromState(si state.Incident) *Incident {
|
|
inc := &Incident{ID: si.ID, Key: si.Key, Severity: si.Severity, Title: si.Title, Detail: si.Detail, Host: si.Host, Href: si.Href,
|
|
Opened: si.Opened, Fired: tp(si.Fired), Notified: tp(si.Notified), Resolved: tp(si.Resolved)}
|
|
if si.AckBy != "" {
|
|
inc.Ack = &Ack{By: si.AckBy, At: si.AckAt}
|
|
}
|
|
var g grouping
|
|
if si.Grouping != "" && json.Unmarshal([]byte(si.Grouping), &g) == nil {
|
|
inc.Affected, inc.Grouped, inc.Map = g.Affected, g.Grouped, g.Map
|
|
}
|
|
return inc
|
|
}
|
|
|
|
// load reads the incidents as stored, replacing what's in memory: at
|
|
// start, on taking over as evaluator, and on copies that aren't the
|
|
// evaluator whenever it says they changed. After a restart that means an
|
|
// upgrade neither forgets an alert nor sends it again.
|
|
func (a *alerter) load() {
|
|
if a.db == nil {
|
|
return
|
|
}
|
|
open, recent, err := a.db.Incidents(historyKeep)
|
|
if err != nil {
|
|
log.Printf("alerts: can't load incidents: %v", err)
|
|
return
|
|
}
|
|
a.mu.Lock()
|
|
defer a.mu.Unlock()
|
|
first := len(a.open) == 0 && len(a.history) == 0
|
|
a.open, a.history = map[string]*Incident{}, nil
|
|
for _, si := range open {
|
|
a.open[si.Key] = fromState(si)
|
|
}
|
|
for _, si := range recent {
|
|
a.history = append(a.history, *fromState(si))
|
|
}
|
|
fmt.Sscan(a.db.Meta("incident_seq"), &a.seq)
|
|
if first && len(open) > 0 {
|
|
log.Printf("alerts: %d open incidents brought back from the database", len(open))
|
|
}
|
|
}
|
|
|
|
// evaluate matches this round's conditions to incidents.
|
|
func (a *alerter) evaluate(now time.Time, conds []Condition) { a.evaluateHeld(now, conds, nil) }
|
|
|
|
// evaluateHeld is evaluate with some hosts in a maintenance window: their
|
|
// problems don't open or fire anything, and incidents already open for them
|
|
// stay exactly as they are (no repeats, and no "resolved" while the host is
|
|
// simply rebooting). When the window closes they're judged normally again.
|
|
func (a *alerter) evaluateHeld(now time.Time, conds []Condition, held map[string]bool) bool {
|
|
a.mu.Lock()
|
|
defer a.mu.Unlock()
|
|
a.changed = false
|
|
defer a.flush()
|
|
seen := map[string]bool{}
|
|
for _, c := range conds {
|
|
if c.Host != "" && held[c.Host] {
|
|
continue
|
|
}
|
|
seen[c.Key] = true
|
|
inc := a.open[c.Key]
|
|
if inc == nil {
|
|
a.seq++
|
|
if a.db != nil {
|
|
a.checkFence(a.db.SetMetaAs("incident_seq", fmt.Sprint(a.seq), a.fence))
|
|
}
|
|
inc = &Incident{ID: fmt.Sprintf("B-%04d", a.seq), Key: c.Key, Opened: now}
|
|
a.open[c.Key] = inc
|
|
}
|
|
before := *inc
|
|
worse := inc.Fired != nil && notify.Rank(c.Severity) > notify.Rank(inc.Severity)
|
|
inc.Severity, inc.Title, inc.Detail, inc.Host, inc.Href, inc.Affected = c.Severity, c.Title, c.Detail, c.Host, c.Href, c.Affected
|
|
inc.Grouped, inc.Map = c.Grouped, c.Map
|
|
switch {
|
|
case inc.Fired == nil && now.Sub(inc.Opened) >= a.cfg.For:
|
|
t := now
|
|
inc.Fired = &t
|
|
a.enqueue(inc, "firing", now)
|
|
case worse:
|
|
if inc.Ack != nil && a.db != nil && !a.fenced {
|
|
_ = a.db.ClearAck(inc.ID)
|
|
}
|
|
inc.Ack = nil // it changed; someone needs to look again
|
|
a.enqueue(inc, "escalated", now)
|
|
case inc.Fired != nil && inc.Severity == "critical" && inc.Ack == nil && now.Sub(*inc.Notified) >= a.cfg.Repeat:
|
|
a.enqueue(inc, "still", now)
|
|
}
|
|
if !inc.same(before) || before.ID == "" || inc.Opened.Equal(now) {
|
|
a.save(inc)
|
|
}
|
|
}
|
|
for key, inc := range a.open {
|
|
if seen[key] || (inc.Host != "" && held[inc.Host]) {
|
|
continue
|
|
}
|
|
delete(a.open, key)
|
|
if inc.Fired == nil {
|
|
if a.db != nil && !a.fenced { // a blip shorter than alerts.for: nobody was told, nothing to keep
|
|
_ = a.db.DeleteIncident(inc.ID)
|
|
a.changed = true
|
|
}
|
|
continue
|
|
}
|
|
t := now
|
|
inc.Resolved = &t
|
|
a.enqueue(inc, "resolved", now)
|
|
a.save(inc)
|
|
a.history = append([]Incident{*inc}, a.history...)
|
|
if len(a.history) > historyKeep {
|
|
a.history = a.history[:historyKeep]
|
|
}
|
|
}
|
|
return a.changed
|
|
}
|
|
|
|
// acknowledge stops the repeats for an open incident. With a database the
|
|
// acknowledgement is written there on its own, so any copy may take it;
|
|
// tell says whether this copy also tells the channels that follow
|
|
// incidents (it's the evaluator, or the only copy).
|
|
func (a *alerter) acknowledge(id, by string, now time.Time, tell bool) (Incident, bool) {
|
|
a.mu.Lock()
|
|
defer a.mu.Unlock()
|
|
for _, inc := range a.open {
|
|
if inc.ID == id {
|
|
if a.db != nil {
|
|
if ok, err := a.db.AckIncident(id, by, now); err != nil || !ok {
|
|
return Incident{}, false
|
|
}
|
|
}
|
|
inc.Ack = &Ack{By: by, At: now}
|
|
if tell {
|
|
a.tellFollowers(inc, "acknowledged")
|
|
a.flush()
|
|
}
|
|
return *inc, true
|
|
}
|
|
}
|
|
return Incident{}, false
|
|
}
|
|
|
|
// acknowledged takes in an acknowledgement another copy made, as stored,
|
|
// and, when tell, tells the channels that follow incidents.
|
|
func (a *alerter) acknowledged(id string, tell bool) {
|
|
si, err := a.db.Incident(id)
|
|
if err != nil || si.AckBy == "" {
|
|
return
|
|
}
|
|
a.mu.Lock()
|
|
defer a.mu.Unlock()
|
|
for _, inc := range a.open {
|
|
if inc.ID == id && inc.Ack == nil {
|
|
inc.Ack = &Ack{By: si.AckBy, At: si.AckAt}
|
|
if tell {
|
|
a.tellFollowers(inc, "acknowledged")
|
|
a.flush()
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// snapshot lists open incidents (critical first, then oldest) and recent history.
|
|
// firing says whether the condition with this key already has an incident
|
|
// people were told about.
|
|
func (a *alerter) firing(key string) bool {
|
|
a.mu.Lock()
|
|
defer a.mu.Unlock()
|
|
inc := a.open[key]
|
|
return inc != nil && inc.Fired != nil
|
|
}
|
|
|
|
func (a *alerter) snapshot() (open, recent []Incident) {
|
|
a.mu.Lock()
|
|
defer a.mu.Unlock()
|
|
open = []Incident{}
|
|
for _, inc := range a.open {
|
|
open = append(open, *inc)
|
|
}
|
|
sort.Slice(open, func(i, j int) bool {
|
|
if ri, rj := notify.Rank(open[i].Severity), notify.Rank(open[j].Severity); ri != rj {
|
|
return ri > rj
|
|
}
|
|
return open[i].Opened.Before(open[j].Opened)
|
|
})
|
|
n := min(len(a.history), 50)
|
|
recent = append([]Incident{}, a.history[:n]...)
|
|
return open, recent
|
|
}
|
|
|
|
// run delivers queued notices until ctx ends.
|
|
func (a *alerter) run(ctx context.Context) {
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case j := <-a.queue:
|
|
for _, ch := range j.chans {
|
|
a.deliver(ctx, ch, j.n)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (a *alerter) deliver(ctx context.Context, ch notify.Channel, n notify.Notice) {
|
|
for attempt := 0; ; attempt++ {
|
|
sctx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
|
err := ch.Send(sctx, n)
|
|
cancel()
|
|
if err == nil {
|
|
log.Printf("alerts: sent %s %s (%s) to %s", n.Kind, n.ID, n.Severity, ch.ID())
|
|
return
|
|
}
|
|
if attempt >= len(a.retry) {
|
|
log.Printf("alerts: gave up on %s for %s %s: %v", ch.ID(), n.Kind, n.ID, err)
|
|
return
|
|
}
|
|
log.Printf("alerts: %v; trying again in %s", err, a.retry[attempt])
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-time.After(a.retry[attempt]):
|
|
}
|
|
}
|
|
}
|
|
|
|
// test sends a test notice right away to every channel, or to the one
|
|
// called only, and reports each result.
|
|
func (a *alerter) test(ctx context.Context, only string) []map[string]string {
|
|
out := []map[string]string{}
|
|
for _, ch := range a.chans() {
|
|
if only != "" && ch.ID() != only {
|
|
continue
|
|
}
|
|
sctx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
|
err := ch.Send(sctx, notify.Notice{Kind: "test", Severity: "warning", Title: "Test", Since: time.Now().In(a.zone()), URL: a.cfg.PublicURL})
|
|
cancel()
|
|
res := map[string]string{"id": ch.ID(), "type": ch.Type(), "result": "sent"}
|
|
if err != nil {
|
|
res["result"] = err.Error()
|
|
}
|
|
log.Printf("alerts: test to %s: %s", ch.ID(), res["result"])
|
|
out = append(out, res)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// --- HTTP ---
|
|
|
|
// evaluate runs one round of the alerts and says whether any incident
|
|
// changed.
|
|
func (srv *Server) evaluate(now time.Time) bool {
|
|
// Right after a start nothing has reported yet, and incidents brought
|
|
// back from the database would look fixed: wait until every agent has
|
|
// had time to report before judging anything.
|
|
if now.Sub(srv.started) < srv.cfg.StaleAfter {
|
|
return false
|
|
}
|
|
_, conds := srv.view(now)
|
|
conds = srv.groupConditions(conds, srv.alerts.firing)
|
|
return srv.alerts.evaluateHeld(now, conds, srv.heldHosts(now))
|
|
}
|
|
|
|
// IncidentView is an open incident as the Alerts page shows it: with the
|
|
// last log lines when it's about a container that has them. Logs stay in
|
|
// Cairn OBS: notices carry the link, not the lines.
|
|
type IncidentView struct {
|
|
Incident
|
|
Logs []string `json:"logs,omitempty"`
|
|
}
|
|
|
|
func (srv *Server) handleIncidents(w http.ResponseWriter, r *http.Request) {
|
|
snapOpen, recent := srv.alerts.snapshot()
|
|
hosts, _ := srv.view(time.Now())
|
|
logs := map[string][]string{}
|
|
for _, h := range hosts {
|
|
if !srv.seesAll(r) {
|
|
break // log lines are for admins and operators
|
|
}
|
|
for _, row := range h.rows {
|
|
if len(row.Logs) > 0 {
|
|
logs["container/"+h.Name+"/"+row.Name] = row.Logs
|
|
}
|
|
}
|
|
}
|
|
open := make([]IncidentView, 0, len(snapOpen))
|
|
for _, inc := range snapOpen {
|
|
open = append(open, IncidentView{Incident: inc, Logs: logs[inc.Key]})
|
|
}
|
|
chans := []map[string]string{}
|
|
for _, ch := range srv.alerts.chans() {
|
|
chans = append(chans, map[string]string{"id": ch.ID(), "type": ch.Type(), "min": ch.Min()})
|
|
}
|
|
writeJSON(w, http.StatusOK, map[string]any{
|
|
"open": open, "recent": recent, "channels": chans,
|
|
"for_s": int(srv.alerts.cfg.For / time.Second), "repeat_s": int(srv.alerts.cfg.Repeat / time.Second), "stored": srv.db != nil,
|
|
"maintenance": srv.maintenanceView(time.Now()), "maintenance_source": srv.maintenanceSource(), "channels_source": srv.channelsSource(),
|
|
})
|
|
}
|
|
|
|
func (srv *Server) handleAck(w http.ResponseWriter, r *http.Request) {
|
|
_, s, _ := srv.sessions.get(r)
|
|
lead := srv.leading()
|
|
inc, ok := srv.alerts.acknowledge(r.PathValue("id"), s.User, time.Now(), lead)
|
|
if !ok {
|
|
writeJSON(w, http.StatusNotFound, map[string]string{"error": "no open incident with that id"})
|
|
return
|
|
}
|
|
if !lead {
|
|
srv.publish(noticeAck, inc.ID) // the evaluator tells the channels that follow incidents
|
|
}
|
|
srv.audit(s.User, "acknowledged", inc.ID+": "+inc.Title)
|
|
writeJSON(w, http.StatusOK, inc)
|
|
}
|
|
|
|
func (srv *Server) handleAlertTest(w http.ResponseWriter, r *http.Request) {
|
|
_, s, _ := srv.sessions.get(r)
|
|
only := r.URL.Query().Get("channel")
|
|
srv.audit(s.User, "sent a test alert", only)
|
|
writeJSON(w, http.StatusOK, map[string]any{"channels": srv.alerts.test(r.Context(), only)})
|
|
}
|
|
|
|
// heldHosts are the hosts in a maintenance window right now.
|
|
func (srv *Server) heldHosts(now time.Time) map[string]bool {
|
|
held := map[string]bool{}
|
|
ws := srv.windows()
|
|
for i := range ws {
|
|
w := &ws[i]
|
|
if w.Active(now) {
|
|
for _, h := range w.Hosts {
|
|
held[h] = true
|
|
}
|
|
}
|
|
}
|
|
return held
|
|
}
|
|
|
|
// maintenanceView is what the Alerts page shows about the windows.
|
|
func (srv *Server) maintenanceView(now time.Time) []map[string]any {
|
|
out := []map[string]any{}
|
|
ws := srv.windows()
|
|
for i := range ws {
|
|
w := &ws[i]
|
|
out = append(out, map[string]any{"name": w.Name, "hosts": w.Hosts, "daily": w.Daily, "timezone": w.Timezone,
|
|
"duration_s": int(w.Duration / time.Second), "active": w.Active(now), "next": w.Next(now)})
|
|
}
|
|
return out
|
|
}
|