From 93b665dfbde2bdc128a5d798e04df86186cf5504 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Mon, 16 Nov 2026 08:38:52 -0700 Subject: [PATCH] Overview: last visit, service strips, a watch list, incidents on attention items What the new dashboard needs from obs-beacond (#70): - today: incidents opened or closed since this person's previous visit (a visit ends after 30 minutes away; none on record, or a demo visitor: the last 12 hours), with counts of opened, resolved, fixed on their own and alerts that went out - services: each service's last 24 hours as 48 half-hour cells (the worst status in each) and its uptime over 24 hours and 30 days; GET /api/v1/services adds its parts, needs and what needs it - watch: disks filling within 30 days, hosts busier than usual for the hour, hosts started in the last day - attention items carry their key, what they take down and their open incident, so the dashboard can offer "I'm on it" History is read at most once a minute (strips), every ten (30 days, the watch list), however many dashboards are open. --- docs/formats.md | 30 ++ internal/server/api.go | 13 +- internal/server/server.go | 3 + internal/server/today.go | 527 +++++++++++++++++++++++++++++++++ internal/server/today_test.go | 153 ++++++++++ internal/state/history.go | 14 +- internal/state/history_test.go | 12 + 7 files changed, 747 insertions(+), 5 deletions(-) create mode 100644 internal/server/today.go create mode 100644 internal/server/today_test.go diff --git a/docs/formats.md b/docs/formats.md index b09459f..c70d2de 100644 --- a/docs/formats.md +++ b/docs/formats.md @@ -726,6 +726,36 @@ expiring within 30 days, disks filling within 90 and mail. Email gets the whole text, chat a line and a link. It's sent once per week even across restarts; admins can preview it and send it by hand on the Reports page. +### The dashboard's briefing + +The same history feeds the dashboard. `GET /api/v1/overview` carries, +besides the readings: + +- `today`: what happened since this person last looked. A visit ends after + 30 minutes without the dashboard open, and the end of the last one is + `since` (`last_visit: true`); with none on record, and for demo visitors, + who share one sign-in, it's the last 12 hours. `events` are the newest + eight incidents that opened or closed since then; `opened`, `resolved`, + `fixed_itself` (closed with nobody having acknowledged it) and `told` + (alerts that went out) count them all. Each person's last visit is kept + in the database's meta table as `visit/`. +- `services`: each service's state and its last 24 hours as 48 half-hour + cells, the worst status in each (empty where nothing is known), with + uptime over 24 hours and 30 days. `GET /api/v1/services` gives the same + with each service's parts, what it needs and what needs it. +- `watch`, the few hosts heading somewhere: disks that fill within 30 days + at the past week's rate (50% full at least), hosts busier than usual for + the hour (CPU at least twice and 15 points over that hour's average in + the past week, memory 15 points over and 60% at least; three days of + that hour first), and hosts that started in the last day. +- each `attention` item's `key`, what else it takes down (`affected`) and, + once there is one, its open `incident` with who acknowledged it, so the + dashboard can offer "I'm on it". + +The strips and uptimes are read from history once a minute (30 days every +ten), the watch list every ten minutes, whatever the number of dashboards +open. + ## checks.yaml: active checks `checks` in obs-beacond.yaml points at it. obs-beacond re-reads it when it changes diff --git a/internal/server/api.go b/internal/server/api.go index 7fbd1f9..938ca71 100644 --- a/internal/server/api.go +++ b/internal/server/api.go @@ -181,6 +181,10 @@ type Attention struct { Detail string `json:"detail"` Href string `json:"href"` Host string `json:"host"` + Key string `json:"key,omitempty"` + Affected string `json:"affected,omitempty"` // what else it takes down, from root-cause grouping + // Incident is the open incident it is, once there is one (today.go). + Incident *AttnIncident `json:"incident,omitempty"` } func nonNil(s []float64) []float64 { @@ -399,13 +403,13 @@ func attention(conds []Condition) []Attention { if shown[c.Host] == 3 { if !folded[c.Host] { folded[c.Host] = true - out = append(out, Attention{"warning", fmt.Sprintf("%d more containers on %s need a look", total[c.Host]-3, c.Host), "", "/containers", c.Host}) + out = append(out, Attention{Severity: "warning", Title: fmt.Sprintf("%d more containers on %s need a look", total[c.Host]-3, c.Host), Href: "/containers", Host: c.Host}) } continue } shown[c.Host]++ } - out = append(out, Attention{c.Severity, c.Title, c.Detail, c.Href, c.Host}) + out = append(out, Attention{Severity: c.Severity, Title: c.Title, Detail: c.Detail, Href: c.Href, Host: c.Host, Key: c.Key, Affected: c.Affected}) } return out } @@ -419,6 +423,7 @@ func (srv *Server) handleOverview(w http.ResponseWriter, r *http.Request) { sum.add(h.Containers) } open, _ := srv.alerts.snapshot() + attn = withIncidents(attn, open) firing, unacked := 0, 0 for _, inc := range open { if inc.Fired != nil { @@ -447,6 +452,10 @@ func (srv *Server) handleOverview(w http.ResponseWriter, r *http.Request) { "attention": attn, "containers": sum, "clusters": srv.clusterViews(now, conds), "alerts": map[string]int{"firing": firing, "unacknowledged": unacked}, "features": srv.features(), + // The dashboard's briefing (today.go): what happened since this + // person last looked, each service's last day, and the hosts + // heading somewhere. + "today": srv.todayFor(r, now), "services": srv.serviceBriefs(now), "watch": srv.watch(hosts, now), }) } diff --git a/internal/server/server.go b/internal/server/server.go index 33f711e..d6b31c4 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -43,6 +43,8 @@ type Server struct { started time.Time alerts *alerter db *state.DB // nil when no database is configured + visits visitBook // each person's previous visit (today.go) + trends trendCache // service strips and the watch list, from history (today.go) atls *agentTLS // nil when the agent listener is plain HTTP acme *autocert.Manager // nil without tls.acme joinedMu sync.RWMutex @@ -425,6 +427,7 @@ func (srv *Server) Handler() http.Handler { mux.HandleFunc("GET /agent-packages/{file}", srv.handleAgentPackage) mux.HandleFunc("GET /api/v1/map/network", srv.needSession(srv.handleMapNetwork)) mux.HandleFunc("GET /api/v1/map/services", srv.needSession(srv.handleMapServices)) + mux.HandleFunc("GET /api/v1/services", srv.needSession(srv.handleServices)) mux.HandleFunc("GET /api/v1/map/find", srv.needSession(srv.handleMapFind)) mux.HandleFunc("GET /api/v1/map/found", srv.needSession(srv.handleMapFound)) mux.HandleFunc("GET /api/v1/map/corrections", srv.needSession(srv.handleMapCorrections)) diff --git a/internal/server/today.go b/internal/server/today.go new file mode 100644 index 0000000..7280382 --- /dev/null +++ b/internal/server/today.go @@ -0,0 +1,527 @@ +package server + +import ( + "fmt" + "net/http" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/cairnobs/cairnobs/internal/state" +) + +// What the dashboard's briefing needs beyond the readings (#69, #70): +// when each person last looked and what happened since, every service's +// last 24 hours, and a watch list of the few hosts heading somewhere. The +// slow parts come from history at most every few minutes, not on every +// poll: every open dashboard reads the overview every few seconds. + +const ( + visitGap = 30 * time.Minute // a visit ends after this long without the page open + sinceDefault = 12 * time.Hour // "since you last looked" with no last visit on record + stripStep = 30 * time.Minute // one bar of a service's strip + stripCells = 48 // a day of them + todayEvents = 8 // events listed; the counts cover all + svcRefresh = time.Minute + svcMonthEvery = 10 * time.Minute + watchRefresh = 10 * time.Minute + watchDays = 7 + watchMax = 5 +) + +// visitBook keeps each person's previous visit. A visit ends after +// visitGap without the page open; the end of the last one is what "since +// you last looked" counts from. Written to the database at most once a +// minute a person, so it survives a restart. +type visitBook struct { + mu sync.Mutex + m map[string]*visitMark +} + +type visitMark struct{ prev, last, saved time.Time } + +// visited notes that user has the dashboard open now and returns the end of +// their previous visit: zero when there's none on record. +func (srv *Server) visited(user string, now time.Time) time.Time { + b := &srv.visits + b.mu.Lock() + defer b.mu.Unlock() + if b.m == nil { + b.m = map[string]*visitMark{} + } + v := b.m[user] + if v == nil { + v = &visitMark{} + if srv.db != nil { + v.prev, v.last = parseVisit(srv.db.Meta("visit/" + user)) + v.saved = v.last + } + b.m[user] = v + } + if !v.last.IsZero() && now.Sub(v.last) > visitGap { + v.prev = v.last + } + v.last = now + if srv.db != nil && now.Sub(v.saved) >= time.Minute { + v.saved = now + _ = srv.db.SetMeta("visit/"+user, fmt.Sprintf("%d %d", unixOrZero(v.prev), unixOrZero(v.last))) + } + return v.prev +} + +func unixOrZero(t time.Time) int64 { + if t.IsZero() { + return 0 + } + return t.Unix() +} + +func parseVisit(s string) (prev, last time.Time) { + a, b, _ := strings.Cut(s, " ") + at := func(x string) time.Time { + n, err := strconv.ParseInt(x, 10, 64) + if err != nil || n <= 0 { + return time.Time{} + } + return time.Unix(n, 0) + } + return at(a), at(b) +} + +// Today is what happened since the person last looked. +type Today struct { + Since time.Time `json:"since"` + LastVisit bool `json:"last_visit"` // Since is their previous visit, not the default 12 hours + // Events are the newest incidents that opened or closed since, newest first. + Events []TodayEvent `json:"events"` + Opened int `json:"opened"` + Resolved int `json:"resolved"` + FixedItself int `json:"fixed_itself"` // resolved with nobody having acknowledged it + Told int `json:"told"` // alerts that went out to people +} + +// TodayEvent is one incident in Today. +type TodayEvent struct { + ID string `json:"id"` + Severity string `json:"severity"` + Title string `json:"title"` + Host string `json:"host,omitempty"` + Href string `json:"href,omitempty"` + At time.Time `json:"at"` // when it opened, or closed when it opened before Since + Opened time.Time `json:"opened"` + Resolved *time.Time `json:"resolved,omitempty"` + AckBy string `json:"ack_by,omitempty"` + Told bool `json:"told"` +} + +// today sums up the incidents that opened or closed since since. +func today(open, recent []Incident, since time.Time) Today { + t := Today{Since: since, Events: []TodayEvent{}} + for _, inc := range append(append([]Incident{}, open...), recent...) { + opened := !inc.Opened.Before(since) + closed := inc.Resolved != nil && !inc.Resolved.Before(since) + if !opened && !closed { + continue + } + e := TodayEvent{ID: inc.ID, Severity: inc.Severity, Title: inc.Title, Host: inc.Host, Href: inc.Href, At: inc.Opened, Opened: inc.Opened, Resolved: inc.Resolved} + if !opened { + e.At = *inc.Resolved + } + if inc.Ack != nil { + e.AckBy = inc.Ack.By + } + if inc.Fired != nil && !inc.Fired.Before(since) { + e.Told = true + t.Told++ + } + if opened { + t.Opened++ + } + if closed { + t.Resolved++ + if inc.Ack == nil { + t.FixedItself++ + } + } + t.Events = append(t.Events, e) + } + sort.SliceStable(t.Events, func(i, j int) bool { + if !t.Events[i].At.Equal(t.Events[j].At) { + return t.Events[i].At.After(t.Events[j].At) + } + return idLess(t.Events[j].ID, t.Events[i].ID) + }) + if len(t.Events) > todayEvents { + t.Events = t.Events[:todayEvents] + } + return t +} + +// todayFor is Today for the person asking. A demo visitor shares one +// sign-in with everyone else, so they get the last 12 hours. +func (srv *Server) todayFor(r *http.Request, now time.Time) Today { + since, last := now.Add(-sinceDefault), false + if _, s, ok := srv.sessions.get(r); ok && !s.demo() && s.User != "" { + if prev := srv.visited(s.User, now); !prev.IsZero() { + since, last = prev, true + } + } + open, recent := srv.alerts.snapshot() + t := today(open, recent, since) + t.LastVisit = last + return t +} + +// ServiceHistory is a service's recent past. +type ServiceHistory struct { + // Cells are the last 24 hours, half an hour each, oldest first: the + // worst status in each, or "" when nothing is known. + Cells []string `json:"cells"` + Pct24 float64 `json:"pct_24h"` // up, of the time known; -1 when nothing is + Pct30 float64 `json:"pct_30d"` +} + +// strips gives each entity's worst status in each of n steps from from. +// changes are oldest first, with each entity's status as from began. +func strips(changes []state.Change, from time.Time, n int, step time.Duration) map[string][]string { + by := map[string][]state.Change{} + for _, c := range changes { + by[c.Entity] = append(by[c.Entity], c) + } + out := map[string][]string{} + for e, cs := range by { + cells := make([]string, n) + cur, j := "", 0 + for i := range n { + t0 := from.Add(time.Duration(i) * step) + t1 := t0.Add(step) + for j < len(cs) && !cs[j].At.After(t0) { + cur = cs[j].Status + j++ + } + w := cur + for k := j; k < len(cs) && cs[k].At.Before(t1); k++ { + w = cellWorst(w, cs[k].Status) + } + if w == "unknown" { + w = "" + } + cells[i] = w + } + out[e] = cells + } + return out +} + +// cellWorst is the worse of two statuses, where not knowing loses to +// anything known: a half hour obs-beacond was down for part of shows what +// it saw in the rest. +func cellWorst(a, b string) string { + if a == "" || a == "unknown" { + return b + } + if b == "" || b == "unknown" { + return a + } + return worst(a, b) +} + +// trendCache holds what comes from history, refreshed now and then. +type trendCache struct { + mu sync.Mutex + svcAt time.Time + monthAt time.Time + svc map[string]ServiceHistory // by service id + month map[string]float64 + watchAt time.Time + filling []WatchDisk + usual map[string]*[24]usualHour // host → its hours of the day over the past week + hoursErr error +} + +type usualHour struct { + n int + cpu, mem float64 +} + +// serviceHistory is every service's ServiceHistory, by id. +func (srv *Server) serviceHistory(now time.Time) map[string]ServiceHistory { + c := &srv.trends + c.mu.Lock() + defer c.mu.Unlock() + if srv.db == nil { + return map[string]ServiceHistory{} + } + if c.svc != nil && now.Sub(c.svcAt) < svcRefresh { + return c.svc + } + if c.month == nil || now.Sub(c.monthAt) >= svcMonthEvery { + c.month = map[string]float64{} + if ch, err := srv.db.StatusChangesOf("svc/", now.Add(-30*24*time.Hour), now); err == nil { + for e, u := range integrate(ch, now.Add(-30*24*time.Hour), now) { + c.month[e] = u.Pct + } + c.monthAt = now + } + } + from := now.Truncate(stripStep).Add(-(stripCells - 1) * stripStep) // the last cell is the half hour now + ch, err := srv.db.StatusChangesOf("svc/", from, now) + if err != nil { + return map[string]ServiceHistory{} + } + day := integrate(ch, now.Add(-24*time.Hour), now) + out := map[string]ServiceHistory{} + for e, cells := range strips(ch, from, stripCells, stripStep) { + id := strings.TrimPrefix(e, "svc/") + h := ServiceHistory{Cells: cells, Pct24: -1, Pct30: -1} + if u := day[e]; u != nil { + h.Pct24 = u.Pct + } + if p, ok := c.month[e]; ok { + h.Pct30 = p + } + out[id] = h + } + c.svc, c.svcAt = out, now + return out +} + +// ServiceBrief is a service as the dashboard shows it. +type ServiceBrief struct { + ID string `json:"id"` + Name string `json:"name"` + Tier string `json:"tier"` + Status string `json:"status"` + Word string `json:"word"` + History ServiceHistory `json:"history"` +} + +// ServiceDetail is a service as the Services page shows it (#71). +type ServiceDetail struct { + ServiceView + NeededBy []string `json:"needed_by"` // the services that need it + History ServiceHistory `json:"history"` +} + +func emptyHistory() ServiceHistory { + return ServiceHistory{Cells: make([]string, stripCells), Pct24: -1, Pct30: -1} +} + +func (srv *Server) serviceBriefs(now time.Time) []ServiceBrief { + views, ok := srv.serviceStates(now) + out := []ServiceBrief{} + if !ok { + return out + } + hist := srv.serviceHistory(now) + for _, v := range views { + h, ok := hist[v.ID] + if !ok { + h = emptyHistory() + } + out = append(out, ServiceBrief{ID: v.ID, Name: v.Name, Tier: v.Tier, Status: v.Status, Word: v.Word, History: h}) + } + return out +} + +func (srv *Server) handleServices(w http.ResponseWriter, r *http.Request) { + now := time.Now() + views, ok := srv.serviceStates(now) + out := []ServiceDetail{} + if ok { + hist := srv.serviceHistory(now) + by := map[string][]string{} + for _, v := range views { + for _, n := range v.Needs { + by[n] = append(by[n], v.ID) + } + } + for _, v := range views { + h, ok := hist[v.ID] + if !ok { + h = emptyHistory() + } + d := ServiceDetail{ServiceView: v, NeededBy: nonNilSlice(by[v.ID]), History: h} + out = append(out, d) + } + } + writeJSON(w, http.StatusOK, map[string]any{"services": out, "topology": ok}) +} + +// Watch is the few hosts heading somewhere, in place of gauges. +type Watch struct { + Filling []WatchDisk `json:"filling"` // disks that fill within 30 days at the past week's rate + Busier []WatchBusy `json:"busier"` // busier than usual for the hour + Restarted []WatchStart `json:"restarted"` // up less than a day, most recent first + RestartedN int `json:"restarted_n"` // all of them; Restarted is the first few +} + +// WatchDisk is a disk filling up. +type WatchDisk struct { + Host string `json:"host"` + Mount string `json:"mount"` + Pct float64 `json:"pct"` + FullDays float64 `json:"full_days"` +} + +// WatchBusy is a host busier than it usually is at this hour. +type WatchBusy struct { + Host string `json:"host"` + What string `json:"what"` // cpu or memory + Now float64 `json:"now"` + Usual float64 `json:"usual"` +} + +// WatchStart is a host that started in the last day. +type WatchStart struct { + Host string `json:"host"` + UpS float64 `json:"up_s"` +} + +// watch is the watch list for hosts as they are now. +func (srv *Server) watch(hosts []HostView, now time.Time) Watch { + w := Watch{Filling: []WatchDisk{}, Busier: []WatchBusy{}, Restarted: []WatchStart{}} + live := map[string]bool{} + for _, h := range hosts { + live[h.Name] = true + } + filling, usual := srv.weekTrends(now) + for _, d := range filling { + if live[d.Host] && len(w.Filling) < watchMax { + w.Filling = append(w.Filling, d) + } + } + w.Busier = busier(hosts, usual, now) + w.Restarted, w.RestartedN = restarted(hosts) + return w +} + +// weekTrends is the past week's disks that are filling, soonest first, and +// each host's usual CPU and memory by hour of the day. +func (srv *Server) weekTrends(now time.Time) ([]WatchDisk, map[string]*[24]usualHour) { + c := &srv.trends + c.mu.Lock() + defer c.mu.Unlock() + if srv.db == nil { + return nil, nil + } + if c.usual != nil && now.Sub(c.watchAt) < watchRefresh { + return c.filling, c.usual + } + c.watchAt = now // a failing read isn't retried on every poll + hours, err := srv.db.HostHours(now.Add(-watchDays*24*time.Hour), now) + if err != nil { + c.hoursErr = err + return c.filling, c.usual + } + c.filling, c.usual = fillingDisks(hours), usualHours(hours) + return c.filling, c.usual +} + +func fillingDisks(hours []state.HostHour) []WatchDisk { + disks, _ := capacity(hours, time.UTC) + out := []WatchDisk{} + for _, d := range disks { + if d.FullDays != nil && *d.FullDays <= 30 && d.Pct >= 50 { + out = append(out, WatchDisk{Host: d.Host, Mount: d.Mount, Pct: d.Pct, FullDays: *d.FullDays}) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].FullDays < out[j].FullDays }) + return out +} + +func usualHours(hours []state.HostHour) map[string]*[24]usualHour { + out := map[string]*[24]usualHour{} + for _, h := range hours { + u := out[h.Host] + if u == nil { + u = &[24]usualHour{} + out[h.Host] = u + } + k := h.Hour.UTC().Hour() + u[k].n++ + u[k].cpu += h.CPU + u[k].mem += h.Mem + } + return out +} + +// busier is the hosts noticeably busier than they usually are at this +// hour of the day: CPU at least twice usual and 15 points up (and 25% at +// least), or memory 15 points up and 60% at least. Three days of that hour +// before anything is usual. +func busier(hosts []HostView, usual map[string]*[24]usualHour, now time.Time) []WatchBusy { + out := []WatchBusy{} + k := now.UTC().Hour() + for _, h := range hosts { + u := usual[h.Name] + if h.Silent || u == nil || u[k].n < 3 { + continue + } + cpu, mem := u[k].cpu/float64(u[k].n), u[k].mem/float64(u[k].n) + if h.CPU >= 25 && h.CPU >= 2*cpu && h.CPU-cpu >= 15 { + out = append(out, WatchBusy{Host: h.Name, What: "cpu", Now: h.CPU, Usual: cpu}) + } + if h.MemPct >= 60 && h.MemPct-mem >= 15 { + out = append(out, WatchBusy{Host: h.Name, What: "memory", Now: h.MemPct, Usual: mem}) + } + } + sort.Slice(out, func(i, j int) bool { + di, dj := out[i].Now-out[i].Usual, out[j].Now-out[j].Usual + if di != dj { + return di > dj + } + return out[i].Host < out[j].Host + }) + if len(out) > watchMax { + out = out[:watchMax] + } + return out +} + +// restarted is the hosts up less than a day, most recent first, and how +// many there are. +func restarted(hosts []HostView) ([]WatchStart, int) { + out := []WatchStart{} + for _, h := range hosts { + if !h.Silent && h.Uptime > 0 && h.Uptime < 24*3600 { + out = append(out, WatchStart{Host: h.Name, UpS: h.Uptime}) + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].UpS != out[j].UpS { + return out[i].UpS < out[j].UpS + } + return out[i].Host < out[j].Host + }) + n := len(out) + if n > watchMax { + out = out[:watchMax] + } + return out, n +} + +// AttnIncident is the open incident behind an attention item, so the +// dashboard can offer "I'm on it" (the acknowledgement). +type AttnIncident struct { + ID string `json:"id"` + Opened time.Time `json:"opened"` + Fired *time.Time `json:"fired"` + Ack *Ack `json:"ack"` +} + +// withIncidents names each attention item's open incident, by key. +func withIncidents(attn []Attention, open []Incident) []Attention { + by := map[string]Incident{} + for _, inc := range open { + by[inc.Key] = inc + } + for i, a := range attn { + if inc, ok := by[a.Key]; ok && a.Key != "" { + attn[i].Incident = &AttnIncident{ID: inc.ID, Opened: inc.Opened, Fired: inc.Fired, Ack: inc.Ack} + } + } + return attn +} diff --git a/internal/server/today_test.go b/internal/server/today_test.go new file mode 100644 index 0000000..62b35e0 --- /dev/null +++ b/internal/server/today_test.go @@ -0,0 +1,153 @@ +package server + +import ( + "strings" + "testing" + "time" + + "github.com/cairnobs/cairnobs/internal/state" +) + +func TestStrips(t *testing.T) { + t0 := time.Date(2026, 11, 16, 0, 0, 0, 0, time.UTC) + at := func(m int) time.Time { return t0.Add(time.Duration(m) * time.Minute) } + got := strips([]state.Change{ + {Entity: "svc/a", At: at(-90), Status: "good"}, // as the day began + {Entity: "svc/a", At: at(40), Status: "critical"}, // a blip inside cell 1 + {Entity: "svc/a", At: at(45), Status: "good"}, // + {Entity: "svc/a", At: at(60), Status: "warning"}, // from cell 2's start + {Entity: "svc/a", At: at(120), Status: "unknown"}, // obs-beacond down + {Entity: "svc/a", At: at(170), Status: "good"}, // back in cell 5 + {Entity: "svc/b", At: at(100), Status: "good"}, // first known in cell 3 + }, t0, 6, 30*time.Minute) + want := map[string]string{ + "svc/a": "good critical warning warning good", + "svc/b": " good good good", + } + for e, w := range want { + if g := strings.Join(got[e], " "); g != w { + t.Errorf("%s = %q, want %q", e, g, w) + } + } +} + +func TestToday(t *testing.T) { + since := time.Date(2026, 11, 16, 2, 0, 0, 0, time.UTC) + at := func(h float64) *time.Time { x := since.Add(time.Duration(h * float64(time.Hour))); return &x } + open := []Incident{ + {ID: "B-10", Severity: "critical", Title: "drive failed", Opened: *at(4.5), Fired: at(4.5)}, + {ID: "B-9", Severity: "warning", Title: "leader moved", Opened: *at(3.8), Ack: &Ack{By: "sam", At: *at(4)}}, + {ID: "B-2", Severity: "warning", Title: "old and still open", Opened: *at(-30)}, + } + recent := []Incident{ + {ID: "B-8", Severity: "warning", Title: "slow shop", Opened: *at(1), Resolved: at(1.2)}, + {ID: "B-7", Severity: "critical", Title: "closed overnight", Opened: *at(-1), Fired: at(-0.9), Resolved: at(0.5), Ack: &Ack{By: "ana"}}, + {ID: "B-1", Severity: "warning", Title: "long gone", Opened: *at(-20), Resolved: at(-19)}, + } + got := today(open, recent, since) + var ids []string + for _, e := range got.Events { + ids = append(ids, e.ID) + } + if j := strings.Join(ids, " "); j != "B-10 B-9 B-8 B-7" { + t.Errorf("events = %s", j) + } + if got.Opened != 3 || got.Resolved != 2 || got.FixedItself != 1 || got.Told != 1 { + t.Errorf("counts: opened %d resolved %d fixed itself %d told %d", got.Opened, got.Resolved, got.FixedItself, got.Told) + } + if e := got.Events[3]; !e.At.Equal(*at(0.5)) || e.AckBy != "ana" || e.Told { + t.Errorf("B-7 (opened before, closed since) = %+v", e) + } + if e := got.Events[1]; e.AckBy != "sam" { + t.Errorf("B-9 ack = %q", e.AckBy) + } + if empty := today(nil, nil, since); empty.Events == nil { + t.Error("events should be [] not null") + } +} + +func TestVisited(t *testing.T) { + srv := &Server{} + t0 := time.Date(2026, 11, 16, 8, 0, 0, 0, time.UTC) + if p := srv.visited("ana", t0); !p.IsZero() { + t.Errorf("first visit: prev = %s", p) + } + if p := srv.visited("ana", t0.Add(10*time.Minute)); !p.IsZero() { + t.Errorf("same visit: prev = %s", p) + } + // Away longer than the gap: the visit that ended at +10m is the previous one. + if p := srv.visited("ana", t0.Add(5*time.Hour)); !p.Equal(t0.Add(10 * time.Minute)) { + t.Errorf("next visit: prev = %s", p) + } + if p := srv.visited("ana", t0.Add(5*time.Hour+time.Minute)); !p.Equal(t0.Add(10 * time.Minute)) { + t.Errorf("polling keeps the previous visit: prev = %s", p) + } + if p := srv.visited("sam", t0.Add(5*time.Hour)); !p.IsZero() { + t.Errorf("another person: prev = %s", p) + } + if p, l := parseVisit("0 1763280000"); !p.IsZero() || l.Unix() != 1763280000 { + t.Errorf("parseVisit = %s %s", p, l) + } +} + +func TestBusier(t *testing.T) { + now := time.Date(2026, 11, 16, 14, 20, 0, 0, time.UTC) + usual := map[string]*[24]usualHour{} + for _, h := range []string{"calm", "hot", "hungry", "young"} { + u := &[24]usualHour{} + n := 3 + if h == "young" { + n = 2 // two days of 14:00 aren't enough to say what's usual + } + u[14] = usualHour{n: n, cpu: 18 * float64(n), mem: 50 * float64(n)} + usual[h] = u + } + got := busier([]HostView{ + {Name: "calm", CPU: 30, MemPct: 55}, // under twice usual, memory only 5 up + {Name: "hot", CPU: 44, MemPct: 52}, // 44 ≥ 2×18 and 26 up + {Name: "hungry", CPU: 10, MemPct: 81}, // memory 31 up + {Name: "young", CPU: 90, MemPct: 90}, + {Name: "new", CPU: 90}, + }, usual, now) + var s []string + for _, b := range got { + s = append(s, b.Host+"/"+b.What) + } + if j := strings.Join(s, " "); j != "hungry/memory hot/cpu" { + t.Errorf("busier = %s", j) + } +} + +func TestRestartedAndFilling(t *testing.T) { + hosts := []HostView{{Name: "a", Uptime: 3600}, {Name: "b", Uptime: 90000}, {Name: "c", Uptime: 600}, {Name: "d", Uptime: 60, Silent: true}} + got, n := restarted(hosts) + if n != 2 || got[0].Host != "c" || got[1].Host != "a" { + t.Errorf("restarted = %+v (%d)", got, n) + } + t0 := time.Date(2026, 11, 9, 0, 0, 0, 0, time.UTC) + var hours []state.HostHour + for i := range 48 { + h := t0.Add(time.Duration(i) * time.Hour) + hours = append(hours, + state.HostHour{Host: "fills", Hour: h, Disks: []state.DiskSample{{Mount: "/var", Used: uint64(600 + i*2), Total: 1000}}}, // 2 a hour, 306 left: ~6 days + state.HostHour{Host: "steady", Hour: h, Disks: []state.DiskSample{{Mount: "/", Used: 900, Total: 1000}}}, + state.HostHour{Host: "slow", Hour: h, Disks: []state.DiskSample{{Mount: "/", Used: uint64(500 + i/12), Total: 1000}}}, // years + ) + } + f := fillingDisks(hours) + if len(f) != 1 || f[0].Host != "fills" || f[0].FullDays < 6 || f[0].FullDays > 7 { + t.Errorf("filling = %+v", f) + } +} + +func TestWithIncidents(t *testing.T) { + attn := []Attention{{Key: "disk/a//", Title: "disk"}, {Key: "", Title: "folded"}, {Key: "unit/b/x", Title: "no incident yet"}} + open := []Incident{{ID: "B-3", Key: "disk/a//", Ack: &Ack{By: "ana"}}, {ID: "B-4", Key: ""}} + got := withIncidents(attn, open) + if got[0].Incident == nil || got[0].Incident.ID != "B-3" || got[0].Incident.Ack.By != "ana" { + t.Errorf("disk: %+v", got[0].Incident) + } + if got[1].Incident != nil || got[2].Incident != nil { + t.Errorf("others: %+v %+v", got[1].Incident, got[2].Incident) + } +} diff --git a/internal/state/history.go b/internal/state/history.go index 18e6915..78f87ed 100644 --- a/internal/state/history.go +++ b/internal/state/history.go @@ -3,6 +3,7 @@ package state import ( "database/sql" "encoding/json" + "strings" "time" ) @@ -62,12 +63,19 @@ func (d *DB) LatestStatuses() (map[string]string, error) { // StatusChanges is every change in [from, to), plus each entity's last // change before from (its status as the period began), oldest first. func (d *DB) StatusChanges(from, to time.Time) ([]Change, error) { + return d.StatusChangesOf("", from, to) +} + +// StatusChangesOf is StatusChanges for the entities whose names start +// with prefix ("svc/": the services). +func (d *DB) StatusChangesOf(prefix string, from, to time.Time) ([]Change, error) { + like := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(prefix) + "%" rows, err := d.query(` SELECT entity, at, status, `+d.kind.rowOrder()+` AS n FROM status_log s - WHERE at < ? AND at = (SELECT MAX(at) FROM status_log WHERE entity = s.entity AND at < ?) + WHERE entity LIKE ? ESCAPE '\' AND at < ? AND at = (SELECT MAX(at) FROM status_log WHERE entity = s.entity AND at < ?) UNION ALL - SELECT entity, at, status, `+d.kind.rowOrder()+` AS n FROM status_log WHERE at >= ? AND at < ? - ORDER BY at, n`, from.Unix(), from.Unix(), from.Unix(), to.Unix()) + SELECT entity, at, status, `+d.kind.rowOrder()+` AS n FROM status_log WHERE entity LIKE ? ESCAPE '\' AND at >= ? AND at < ? + ORDER BY at, n`, like, from.Unix(), from.Unix(), like, from.Unix(), to.Unix()) if err != nil { return nil, err } diff --git a/internal/state/history_test.go b/internal/state/history_test.go index 1a8d765..8ae2be9 100644 --- a/internal/state/history_test.go +++ b/internal/state/history_test.go @@ -48,6 +48,18 @@ func TestStatusHistory(t *testing.T) { if len(got) != 2 { t.Errorf("after prune: %+v", got) } + // One kind of entity only; the prefix's _ is a letter, not a wildcard. + _ = db.LogStatus("host/mail", "critical", t0.Add(61*24*time.Hour)) + _ = db.LogStatus("svcXmail", "critical", t0.Add(61*24*time.Hour)) + _ = db.LogStatus("svc_mail", "critical", t0.Add(61*24*time.Hour)) + got, _ = db.StatusChangesOf("svc/", t0.Add(61*24*time.Hour), t0.Add(62*24*time.Hour)) + if len(got) != 2 || got[0].Entity[:4] != "svc/" || got[1].Entity[:4] != "svc/" { + t.Errorf("svc/ only: %+v", got) + } + got, _ = db.StatusChangesOf("svc_", t0, t0.Add(62*24*time.Hour)) + if len(got) != 1 || got[0].Entity != "svc_mail" { + t.Errorf("svc_ only: %+v", got) + } } func join(s []string) string { -- 2.54.0