Overview: last visit, service strips, a watch list, incidents on attention items #73
No files matched your search
@@ -726,6 +726,36 @@ expiring within 30 days, disks filling within 90 and mail. Email gets the
|
||||
whole text, chat a line and a link. It's sent once per week even across
|
||||
restarts; admins can preview it and send it by hand on the Reports page.
|
||||
|
||||
### The dashboard's briefing
|
||||
|
||||
The same history feeds the dashboard. `GET /api/v1/overview` carries,
|
||||
besides the readings:
|
||||
|
||||
- `today`: what happened since this person last looked. A visit ends after
|
||||
30 minutes without the dashboard open, and the end of the last one is
|
||||
`since` (`last_visit: true`); with none on record, and for demo visitors,
|
||||
who share one sign-in, it's the last 12 hours. `events` are the newest
|
||||
eight incidents that opened or closed since then; `opened`, `resolved`,
|
||||
`fixed_itself` (closed with nobody having acknowledged it) and `told`
|
||||
(alerts that went out) count them all. Each person's last visit is kept
|
||||
in the database's meta table as `visit/<user>`.
|
||||
- `services`: each service's state and its last 24 hours as 48 half-hour
|
||||
cells, the worst status in each (empty where nothing is known), with
|
||||
uptime over 24 hours and 30 days. `GET /api/v1/services` gives the same
|
||||
with each service's parts, what it needs and what needs it.
|
||||
- `watch`, the few hosts heading somewhere: disks that fill within 30 days
|
||||
at the past week's rate (50% full at least), hosts busier than usual for
|
||||
the hour (CPU at least twice and 15 points over that hour's average in
|
||||
the past week, memory 15 points over and 60% at least; three days of
|
||||
that hour first), and hosts that started in the last day.
|
||||
- each `attention` item's `key`, what else it takes down (`affected`) and,
|
||||
once there is one, its open `incident` with who acknowledged it, so the
|
||||
dashboard can offer "I'm on it".
|
||||
|
||||
The strips and uptimes are read from history once a minute (30 days every
|
||||
ten), the watch list every ten minutes, whatever the number of dashboards
|
||||
open.
|
||||
|
||||
## checks.yaml: active checks
|
||||
|
||||
`checks` in obs-beacond.yaml points at it. obs-beacond re-reads it when it changes
|
||||
|
||||
+11
-2
@@ -181,6 +181,10 @@ type Attention struct {
|
||||
Detail string `json:"detail"`
|
||||
Href string `json:"href"`
|
||||
Host string `json:"host"`
|
||||
Key string `json:"key,omitempty"`
|
||||
Affected string `json:"affected,omitempty"` // what else it takes down, from root-cause grouping
|
||||
// Incident is the open incident it is, once there is one (today.go).
|
||||
Incident *AttnIncident `json:"incident,omitempty"`
|
||||
}
|
||||
|
||||
func nonNil(s []float64) []float64 {
|
||||
@@ -399,13 +403,13 @@ func attention(conds []Condition) []Attention {
|
||||
if shown[c.Host] == 3 {
|
||||
if !folded[c.Host] {
|
||||
folded[c.Host] = true
|
||||
out = append(out, Attention{"warning", fmt.Sprintf("%d more containers on %s need a look", total[c.Host]-3, c.Host), "", "/containers", c.Host})
|
||||
out = append(out, Attention{Severity: "warning", Title: fmt.Sprintf("%d more containers on %s need a look", total[c.Host]-3, c.Host), Href: "/containers", Host: c.Host})
|
||||
}
|
||||
continue
|
||||
}
|
||||
shown[c.Host]++
|
||||
}
|
||||
out = append(out, Attention{c.Severity, c.Title, c.Detail, c.Href, c.Host})
|
||||
out = append(out, Attention{Severity: c.Severity, Title: c.Title, Detail: c.Detail, Href: c.Href, Host: c.Host, Key: c.Key, Affected: c.Affected})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -419,6 +423,7 @@ func (srv *Server) handleOverview(w http.ResponseWriter, r *http.Request) {
|
||||
sum.add(h.Containers)
|
||||
}
|
||||
open, _ := srv.alerts.snapshot()
|
||||
attn = withIncidents(attn, open)
|
||||
firing, unacked := 0, 0
|
||||
for _, inc := range open {
|
||||
if inc.Fired != nil {
|
||||
@@ -447,6 +452,10 @@ func (srv *Server) handleOverview(w http.ResponseWriter, r *http.Request) {
|
||||
"attention": attn, "containers": sum, "clusters": srv.clusterViews(now, conds),
|
||||
"alerts": map[string]int{"firing": firing, "unacknowledged": unacked},
|
||||
"features": srv.features(),
|
||||
// The dashboard's briefing (today.go): what happened since this
|
||||
// person last looked, each service's last day, and the hosts
|
||||
// heading somewhere.
|
||||
"today": srv.todayFor(r, now), "services": srv.serviceBriefs(now), "watch": srv.watch(hosts, now),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -43,6 +43,8 @@ type Server struct {
|
||||
started time.Time
|
||||
alerts *alerter
|
||||
db *state.DB // nil when no database is configured
|
||||
visits visitBook // each person's previous visit (today.go)
|
||||
trends trendCache // service strips and the watch list, from history (today.go)
|
||||
atls *agentTLS // nil when the agent listener is plain HTTP
|
||||
acme *autocert.Manager // nil without tls.acme
|
||||
joinedMu sync.RWMutex
|
||||
@@ -425,6 +427,7 @@ func (srv *Server) Handler() http.Handler {
|
||||
mux.HandleFunc("GET /agent-packages/{file}", srv.handleAgentPackage)
|
||||
mux.HandleFunc("GET /api/v1/map/network", srv.needSession(srv.handleMapNetwork))
|
||||
mux.HandleFunc("GET /api/v1/map/services", srv.needSession(srv.handleMapServices))
|
||||
mux.HandleFunc("GET /api/v1/services", srv.needSession(srv.handleServices))
|
||||
mux.HandleFunc("GET /api/v1/map/find", srv.needSession(srv.handleMapFind))
|
||||
mux.HandleFunc("GET /api/v1/map/found", srv.needSession(srv.handleMapFound))
|
||||
mux.HandleFunc("GET /api/v1/map/corrections", srv.needSession(srv.handleMapCorrections))
|
||||
|
||||
@@ -0,0 +1,527 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/cairnobs/cairnobs/internal/state"
|
||||
)
|
||||
|
||||
// What the dashboard's briefing needs beyond the readings (#69, #70):
|
||||
// when each person last looked and what happened since, every service's
|
||||
// last 24 hours, and a watch list of the few hosts heading somewhere. The
|
||||
// slow parts come from history at most every few minutes, not on every
|
||||
// poll: every open dashboard reads the overview every few seconds.
|
||||
|
||||
const (
|
||||
visitGap = 30 * time.Minute // a visit ends after this long without the page open
|
||||
sinceDefault = 12 * time.Hour // "since you last looked" with no last visit on record
|
||||
stripStep = 30 * time.Minute // one bar of a service's strip
|
||||
stripCells = 48 // a day of them
|
||||
todayEvents = 8 // events listed; the counts cover all
|
||||
svcRefresh = time.Minute
|
||||
svcMonthEvery = 10 * time.Minute
|
||||
watchRefresh = 10 * time.Minute
|
||||
watchDays = 7
|
||||
watchMax = 5
|
||||
)
|
||||
|
||||
// visitBook keeps each person's previous visit. A visit ends after
|
||||
// visitGap without the page open; the end of the last one is what "since
|
||||
// you last looked" counts from. Written to the database at most once a
|
||||
// minute a person, so it survives a restart.
|
||||
type visitBook struct {
|
||||
mu sync.Mutex
|
||||
m map[string]*visitMark
|
||||
}
|
||||
|
||||
type visitMark struct{ prev, last, saved time.Time }
|
||||
|
||||
// visited notes that user has the dashboard open now and returns the end of
|
||||
// their previous visit: zero when there's none on record.
|
||||
func (srv *Server) visited(user string, now time.Time) time.Time {
|
||||
b := &srv.visits
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.m == nil {
|
||||
b.m = map[string]*visitMark{}
|
||||
}
|
||||
v := b.m[user]
|
||||
if v == nil {
|
||||
v = &visitMark{}
|
||||
if srv.db != nil {
|
||||
v.prev, v.last = parseVisit(srv.db.Meta("visit/" + user))
|
||||
v.saved = v.last
|
||||
}
|
||||
b.m[user] = v
|
||||
}
|
||||
if !v.last.IsZero() && now.Sub(v.last) > visitGap {
|
||||
v.prev = v.last
|
||||
}
|
||||
v.last = now
|
||||
if srv.db != nil && now.Sub(v.saved) >= time.Minute {
|
||||
v.saved = now
|
||||
_ = srv.db.SetMeta("visit/"+user, fmt.Sprintf("%d %d", unixOrZero(v.prev), unixOrZero(v.last)))
|
||||
}
|
||||
return v.prev
|
||||
}
|
||||
|
||||
func unixOrZero(t time.Time) int64 {
|
||||
if t.IsZero() {
|
||||
return 0
|
||||
}
|
||||
return t.Unix()
|
||||
}
|
||||
|
||||
func parseVisit(s string) (prev, last time.Time) {
|
||||
a, b, _ := strings.Cut(s, " ")
|
||||
at := func(x string) time.Time {
|
||||
n, err := strconv.ParseInt(x, 10, 64)
|
||||
if err != nil || n <= 0 {
|
||||
return time.Time{}
|
||||
}
|
||||
return time.Unix(n, 0)
|
||||
}
|
||||
return at(a), at(b)
|
||||
}
|
||||
|
||||
// Today is what happened since the person last looked.
|
||||
type Today struct {
|
||||
Since time.Time `json:"since"`
|
||||
LastVisit bool `json:"last_visit"` // Since is their previous visit, not the default 12 hours
|
||||
// Events are the newest incidents that opened or closed since, newest first.
|
||||
Events []TodayEvent `json:"events"`
|
||||
Opened int `json:"opened"`
|
||||
Resolved int `json:"resolved"`
|
||||
FixedItself int `json:"fixed_itself"` // resolved with nobody having acknowledged it
|
||||
Told int `json:"told"` // alerts that went out to people
|
||||
}
|
||||
|
||||
// TodayEvent is one incident in Today.
|
||||
type TodayEvent struct {
|
||||
ID string `json:"id"`
|
||||
Severity string `json:"severity"`
|
||||
Title string `json:"title"`
|
||||
Host string `json:"host,omitempty"`
|
||||
Href string `json:"href,omitempty"`
|
||||
At time.Time `json:"at"` // when it opened, or closed when it opened before Since
|
||||
Opened time.Time `json:"opened"`
|
||||
Resolved *time.Time `json:"resolved,omitempty"`
|
||||
AckBy string `json:"ack_by,omitempty"`
|
||||
Told bool `json:"told"`
|
||||
}
|
||||
|
||||
// today sums up the incidents that opened or closed since since.
|
||||
func today(open, recent []Incident, since time.Time) Today {
|
||||
t := Today{Since: since, Events: []TodayEvent{}}
|
||||
for _, inc := range append(append([]Incident{}, open...), recent...) {
|
||||
opened := !inc.Opened.Before(since)
|
||||
closed := inc.Resolved != nil && !inc.Resolved.Before(since)
|
||||
if !opened && !closed {
|
||||
continue
|
||||
}
|
||||
e := TodayEvent{ID: inc.ID, Severity: inc.Severity, Title: inc.Title, Host: inc.Host, Href: inc.Href, At: inc.Opened, Opened: inc.Opened, Resolved: inc.Resolved}
|
||||
if !opened {
|
||||
e.At = *inc.Resolved
|
||||
}
|
||||
if inc.Ack != nil {
|
||||
e.AckBy = inc.Ack.By
|
||||
}
|
||||
if inc.Fired != nil && !inc.Fired.Before(since) {
|
||||
e.Told = true
|
||||
t.Told++
|
||||
}
|
||||
if opened {
|
||||
t.Opened++
|
||||
}
|
||||
if closed {
|
||||
t.Resolved++
|
||||
if inc.Ack == nil {
|
||||
t.FixedItself++
|
||||
}
|
||||
}
|
||||
t.Events = append(t.Events, e)
|
||||
}
|
||||
sort.SliceStable(t.Events, func(i, j int) bool {
|
||||
if !t.Events[i].At.Equal(t.Events[j].At) {
|
||||
return t.Events[i].At.After(t.Events[j].At)
|
||||
}
|
||||
return idLess(t.Events[j].ID, t.Events[i].ID)
|
||||
})
|
||||
if len(t.Events) > todayEvents {
|
||||
t.Events = t.Events[:todayEvents]
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// todayFor is Today for the person asking. A demo visitor shares one
|
||||
// sign-in with everyone else, so they get the last 12 hours.
|
||||
func (srv *Server) todayFor(r *http.Request, now time.Time) Today {
|
||||
since, last := now.Add(-sinceDefault), false
|
||||
if _, s, ok := srv.sessions.get(r); ok && !s.demo() && s.User != "" {
|
||||
if prev := srv.visited(s.User, now); !prev.IsZero() {
|
||||
since, last = prev, true
|
||||
}
|
||||
}
|
||||
open, recent := srv.alerts.snapshot()
|
||||
t := today(open, recent, since)
|
||||
t.LastVisit = last
|
||||
return t
|
||||
}
|
||||
|
||||
// ServiceHistory is a service's recent past.
|
||||
type ServiceHistory struct {
|
||||
// Cells are the last 24 hours, half an hour each, oldest first: the
|
||||
// worst status in each, or "" when nothing is known.
|
||||
Cells []string `json:"cells"`
|
||||
Pct24 float64 `json:"pct_24h"` // up, of the time known; -1 when nothing is
|
||||
Pct30 float64 `json:"pct_30d"`
|
||||
}
|
||||
|
||||
// strips gives each entity's worst status in each of n steps from from.
|
||||
// changes are oldest first, with each entity's status as from began.
|
||||
func strips(changes []state.Change, from time.Time, n int, step time.Duration) map[string][]string {
|
||||
by := map[string][]state.Change{}
|
||||
for _, c := range changes {
|
||||
by[c.Entity] = append(by[c.Entity], c)
|
||||
}
|
||||
out := map[string][]string{}
|
||||
for e, cs := range by {
|
||||
cells := make([]string, n)
|
||||
cur, j := "", 0
|
||||
for i := range n {
|
||||
t0 := from.Add(time.Duration(i) * step)
|
||||
t1 := t0.Add(step)
|
||||
for j < len(cs) && !cs[j].At.After(t0) {
|
||||
cur = cs[j].Status
|
||||
j++
|
||||
}
|
||||
w := cur
|
||||
for k := j; k < len(cs) && cs[k].At.Before(t1); k++ {
|
||||
w = cellWorst(w, cs[k].Status)
|
||||
}
|
||||
if w == "unknown" {
|
||||
w = ""
|
||||
}
|
||||
cells[i] = w
|
||||
}
|
||||
out[e] = cells
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// cellWorst is the worse of two statuses, where not knowing loses to
|
||||
// anything known: a half hour obs-beacond was down for part of shows what
|
||||
// it saw in the rest.
|
||||
func cellWorst(a, b string) string {
|
||||
if a == "" || a == "unknown" {
|
||||
return b
|
||||
}
|
||||
if b == "" || b == "unknown" {
|
||||
return a
|
||||
}
|
||||
return worst(a, b)
|
||||
}
|
||||
|
||||
// trendCache holds what comes from history, refreshed now and then.
|
||||
type trendCache struct {
|
||||
mu sync.Mutex
|
||||
svcAt time.Time
|
||||
monthAt time.Time
|
||||
svc map[string]ServiceHistory // by service id
|
||||
month map[string]float64
|
||||
watchAt time.Time
|
||||
filling []WatchDisk
|
||||
usual map[string]*[24]usualHour // host → its hours of the day over the past week
|
||||
hoursErr error
|
||||
}
|
||||
|
||||
type usualHour struct {
|
||||
n int
|
||||
cpu, mem float64
|
||||
}
|
||||
|
||||
// serviceHistory is every service's ServiceHistory, by id.
|
||||
func (srv *Server) serviceHistory(now time.Time) map[string]ServiceHistory {
|
||||
c := &srv.trends
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if srv.db == nil {
|
||||
return map[string]ServiceHistory{}
|
||||
}
|
||||
if c.svc != nil && now.Sub(c.svcAt) < svcRefresh {
|
||||
return c.svc
|
||||
}
|
||||
if c.month == nil || now.Sub(c.monthAt) >= svcMonthEvery {
|
||||
c.month = map[string]float64{}
|
||||
if ch, err := srv.db.StatusChangesOf("svc/", now.Add(-30*24*time.Hour), now); err == nil {
|
||||
for e, u := range integrate(ch, now.Add(-30*24*time.Hour), now) {
|
||||
c.month[e] = u.Pct
|
||||
}
|
||||
c.monthAt = now
|
||||
}
|
||||
}
|
||||
from := now.Truncate(stripStep).Add(-(stripCells - 1) * stripStep) // the last cell is the half hour now
|
||||
ch, err := srv.db.StatusChangesOf("svc/", from, now)
|
||||
if err != nil {
|
||||
return map[string]ServiceHistory{}
|
||||
}
|
||||
day := integrate(ch, now.Add(-24*time.Hour), now)
|
||||
out := map[string]ServiceHistory{}
|
||||
for e, cells := range strips(ch, from, stripCells, stripStep) {
|
||||
id := strings.TrimPrefix(e, "svc/")
|
||||
h := ServiceHistory{Cells: cells, Pct24: -1, Pct30: -1}
|
||||
if u := day[e]; u != nil {
|
||||
h.Pct24 = u.Pct
|
||||
}
|
||||
if p, ok := c.month[e]; ok {
|
||||
h.Pct30 = p
|
||||
}
|
||||
out[id] = h
|
||||
}
|
||||
c.svc, c.svcAt = out, now
|
||||
return out
|
||||
}
|
||||
|
||||
// ServiceBrief is a service as the dashboard shows it.
|
||||
type ServiceBrief struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Tier string `json:"tier"`
|
||||
Status string `json:"status"`
|
||||
Word string `json:"word"`
|
||||
History ServiceHistory `json:"history"`
|
||||
}
|
||||
|
||||
// ServiceDetail is a service as the Services page shows it (#71).
|
||||
type ServiceDetail struct {
|
||||
ServiceView
|
||||
NeededBy []string `json:"needed_by"` // the services that need it
|
||||
History ServiceHistory `json:"history"`
|
||||
}
|
||||
|
||||
func emptyHistory() ServiceHistory {
|
||||
return ServiceHistory{Cells: make([]string, stripCells), Pct24: -1, Pct30: -1}
|
||||
}
|
||||
|
||||
func (srv *Server) serviceBriefs(now time.Time) []ServiceBrief {
|
||||
views, ok := srv.serviceStates(now)
|
||||
out := []ServiceBrief{}
|
||||
if !ok {
|
||||
return out
|
||||
}
|
||||
hist := srv.serviceHistory(now)
|
||||
for _, v := range views {
|
||||
h, ok := hist[v.ID]
|
||||
if !ok {
|
||||
h = emptyHistory()
|
||||
}
|
||||
out = append(out, ServiceBrief{ID: v.ID, Name: v.Name, Tier: v.Tier, Status: v.Status, Word: v.Word, History: h})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (srv *Server) handleServices(w http.ResponseWriter, r *http.Request) {
|
||||
now := time.Now()
|
||||
views, ok := srv.serviceStates(now)
|
||||
out := []ServiceDetail{}
|
||||
if ok {
|
||||
hist := srv.serviceHistory(now)
|
||||
by := map[string][]string{}
|
||||
for _, v := range views {
|
||||
for _, n := range v.Needs {
|
||||
by[n] = append(by[n], v.ID)
|
||||
}
|
||||
}
|
||||
for _, v := range views {
|
||||
h, ok := hist[v.ID]
|
||||
if !ok {
|
||||
h = emptyHistory()
|
||||
}
|
||||
d := ServiceDetail{ServiceView: v, NeededBy: nonNilSlice(by[v.ID]), History: h}
|
||||
out = append(out, d)
|
||||
}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"services": out, "topology": ok})
|
||||
}
|
||||
|
||||
// Watch is the few hosts heading somewhere, in place of gauges.
|
||||
type Watch struct {
|
||||
Filling []WatchDisk `json:"filling"` // disks that fill within 30 days at the past week's rate
|
||||
Busier []WatchBusy `json:"busier"` // busier than usual for the hour
|
||||
Restarted []WatchStart `json:"restarted"` // up less than a day, most recent first
|
||||
RestartedN int `json:"restarted_n"` // all of them; Restarted is the first few
|
||||
}
|
||||
|
||||
// WatchDisk is a disk filling up.
|
||||
type WatchDisk struct {
|
||||
Host string `json:"host"`
|
||||
Mount string `json:"mount"`
|
||||
Pct float64 `json:"pct"`
|
||||
FullDays float64 `json:"full_days"`
|
||||
}
|
||||
|
||||
// WatchBusy is a host busier than it usually is at this hour.
|
||||
type WatchBusy struct {
|
||||
Host string `json:"host"`
|
||||
What string `json:"what"` // cpu or memory
|
||||
Now float64 `json:"now"`
|
||||
Usual float64 `json:"usual"`
|
||||
}
|
||||
|
||||
// WatchStart is a host that started in the last day.
|
||||
type WatchStart struct {
|
||||
Host string `json:"host"`
|
||||
UpS float64 `json:"up_s"`
|
||||
}
|
||||
|
||||
// watch is the watch list for hosts as they are now.
|
||||
func (srv *Server) watch(hosts []HostView, now time.Time) Watch {
|
||||
w := Watch{Filling: []WatchDisk{}, Busier: []WatchBusy{}, Restarted: []WatchStart{}}
|
||||
live := map[string]bool{}
|
||||
for _, h := range hosts {
|
||||
live[h.Name] = true
|
||||
}
|
||||
filling, usual := srv.weekTrends(now)
|
||||
for _, d := range filling {
|
||||
if live[d.Host] && len(w.Filling) < watchMax {
|
||||
w.Filling = append(w.Filling, d)
|
||||
}
|
||||
}
|
||||
w.Busier = busier(hosts, usual, now)
|
||||
w.Restarted, w.RestartedN = restarted(hosts)
|
||||
return w
|
||||
}
|
||||
|
||||
// weekTrends is the past week's disks that are filling, soonest first, and
|
||||
// each host's usual CPU and memory by hour of the day.
|
||||
func (srv *Server) weekTrends(now time.Time) ([]WatchDisk, map[string]*[24]usualHour) {
|
||||
c := &srv.trends
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if srv.db == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if c.usual != nil && now.Sub(c.watchAt) < watchRefresh {
|
||||
return c.filling, c.usual
|
||||
}
|
||||
c.watchAt = now // a failing read isn't retried on every poll
|
||||
hours, err := srv.db.HostHours(now.Add(-watchDays*24*time.Hour), now)
|
||||
if err != nil {
|
||||
c.hoursErr = err
|
||||
return c.filling, c.usual
|
||||
}
|
||||
c.filling, c.usual = fillingDisks(hours), usualHours(hours)
|
||||
return c.filling, c.usual
|
||||
}
|
||||
|
||||
func fillingDisks(hours []state.HostHour) []WatchDisk {
|
||||
disks, _ := capacity(hours, time.UTC)
|
||||
out := []WatchDisk{}
|
||||
for _, d := range disks {
|
||||
if d.FullDays != nil && *d.FullDays <= 30 && d.Pct >= 50 {
|
||||
out = append(out, WatchDisk{Host: d.Host, Mount: d.Mount, Pct: d.Pct, FullDays: *d.FullDays})
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].FullDays < out[j].FullDays })
|
||||
return out
|
||||
}
|
||||
|
||||
func usualHours(hours []state.HostHour) map[string]*[24]usualHour {
|
||||
out := map[string]*[24]usualHour{}
|
||||
for _, h := range hours {
|
||||
u := out[h.Host]
|
||||
if u == nil {
|
||||
u = &[24]usualHour{}
|
||||
out[h.Host] = u
|
||||
}
|
||||
k := h.Hour.UTC().Hour()
|
||||
u[k].n++
|
||||
u[k].cpu += h.CPU
|
||||
u[k].mem += h.Mem
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// busier is the hosts noticeably busier than they usually are at this
|
||||
// hour of the day: CPU at least twice usual and 15 points up (and 25% at
|
||||
// least), or memory 15 points up and 60% at least. Three days of that hour
|
||||
// before anything is usual.
|
||||
func busier(hosts []HostView, usual map[string]*[24]usualHour, now time.Time) []WatchBusy {
|
||||
out := []WatchBusy{}
|
||||
k := now.UTC().Hour()
|
||||
for _, h := range hosts {
|
||||
u := usual[h.Name]
|
||||
if h.Silent || u == nil || u[k].n < 3 {
|
||||
continue
|
||||
}
|
||||
cpu, mem := u[k].cpu/float64(u[k].n), u[k].mem/float64(u[k].n)
|
||||
if h.CPU >= 25 && h.CPU >= 2*cpu && h.CPU-cpu >= 15 {
|
||||
out = append(out, WatchBusy{Host: h.Name, What: "cpu", Now: h.CPU, Usual: cpu})
|
||||
}
|
||||
if h.MemPct >= 60 && h.MemPct-mem >= 15 {
|
||||
out = append(out, WatchBusy{Host: h.Name, What: "memory", Now: h.MemPct, Usual: mem})
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
di, dj := out[i].Now-out[i].Usual, out[j].Now-out[j].Usual
|
||||
if di != dj {
|
||||
return di > dj
|
||||
}
|
||||
return out[i].Host < out[j].Host
|
||||
})
|
||||
if len(out) > watchMax {
|
||||
out = out[:watchMax]
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// restarted is the hosts up less than a day, most recent first, and how
|
||||
// many there are.
|
||||
func restarted(hosts []HostView) ([]WatchStart, int) {
|
||||
out := []WatchStart{}
|
||||
for _, h := range hosts {
|
||||
if !h.Silent && h.Uptime > 0 && h.Uptime < 24*3600 {
|
||||
out = append(out, WatchStart{Host: h.Name, UpS: h.Uptime})
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].UpS != out[j].UpS {
|
||||
return out[i].UpS < out[j].UpS
|
||||
}
|
||||
return out[i].Host < out[j].Host
|
||||
})
|
||||
n := len(out)
|
||||
if n > watchMax {
|
||||
out = out[:watchMax]
|
||||
}
|
||||
return out, n
|
||||
}
|
||||
|
||||
// AttnIncident is the open incident behind an attention item, so the
|
||||
// dashboard can offer "I'm on it" (the acknowledgement).
|
||||
type AttnIncident struct {
|
||||
ID string `json:"id"`
|
||||
Opened time.Time `json:"opened"`
|
||||
Fired *time.Time `json:"fired"`
|
||||
Ack *Ack `json:"ack"`
|
||||
}
|
||||
|
||||
// withIncidents names each attention item's open incident, by key.
|
||||
func withIncidents(attn []Attention, open []Incident) []Attention {
|
||||
by := map[string]Incident{}
|
||||
for _, inc := range open {
|
||||
by[inc.Key] = inc
|
||||
}
|
||||
for i, a := range attn {
|
||||
if inc, ok := by[a.Key]; ok && a.Key != "" {
|
||||
attn[i].Incident = &AttnIncident{ID: inc.ID, Opened: inc.Opened, Fired: inc.Fired, Ack: inc.Ack}
|
||||
}
|
||||
}
|
||||
return attn
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/cairnobs/cairnobs/internal/state"
|
||||
)
|
||||
|
||||
func TestStrips(t *testing.T) {
|
||||
t0 := time.Date(2026, 11, 16, 0, 0, 0, 0, time.UTC)
|
||||
at := func(m int) time.Time { return t0.Add(time.Duration(m) * time.Minute) }
|
||||
got := strips([]state.Change{
|
||||
{Entity: "svc/a", At: at(-90), Status: "good"}, // as the day began
|
||||
{Entity: "svc/a", At: at(40), Status: "critical"}, // a blip inside cell 1
|
||||
{Entity: "svc/a", At: at(45), Status: "good"}, //
|
||||
{Entity: "svc/a", At: at(60), Status: "warning"}, // from cell 2's start
|
||||
{Entity: "svc/a", At: at(120), Status: "unknown"}, // obs-beacond down
|
||||
{Entity: "svc/a", At: at(170), Status: "good"}, // back in cell 5
|
||||
{Entity: "svc/b", At: at(100), Status: "good"}, // first known in cell 3
|
||||
}, t0, 6, 30*time.Minute)
|
||||
want := map[string]string{
|
||||
"svc/a": "good critical warning warning good",
|
||||
"svc/b": " good good good",
|
||||
}
|
||||
for e, w := range want {
|
||||
if g := strings.Join(got[e], " "); g != w {
|
||||
t.Errorf("%s = %q, want %q", e, g, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestToday(t *testing.T) {
|
||||
since := time.Date(2026, 11, 16, 2, 0, 0, 0, time.UTC)
|
||||
at := func(h float64) *time.Time { x := since.Add(time.Duration(h * float64(time.Hour))); return &x }
|
||||
open := []Incident{
|
||||
{ID: "B-10", Severity: "critical", Title: "drive failed", Opened: *at(4.5), Fired: at(4.5)},
|
||||
{ID: "B-9", Severity: "warning", Title: "leader moved", Opened: *at(3.8), Ack: &Ack{By: "sam", At: *at(4)}},
|
||||
{ID: "B-2", Severity: "warning", Title: "old and still open", Opened: *at(-30)},
|
||||
}
|
||||
recent := []Incident{
|
||||
{ID: "B-8", Severity: "warning", Title: "slow shop", Opened: *at(1), Resolved: at(1.2)},
|
||||
{ID: "B-7", Severity: "critical", Title: "closed overnight", Opened: *at(-1), Fired: at(-0.9), Resolved: at(0.5), Ack: &Ack{By: "ana"}},
|
||||
{ID: "B-1", Severity: "warning", Title: "long gone", Opened: *at(-20), Resolved: at(-19)},
|
||||
}
|
||||
got := today(open, recent, since)
|
||||
var ids []string
|
||||
for _, e := range got.Events {
|
||||
ids = append(ids, e.ID)
|
||||
}
|
||||
if j := strings.Join(ids, " "); j != "B-10 B-9 B-8 B-7" {
|
||||
t.Errorf("events = %s", j)
|
||||
}
|
||||
if got.Opened != 3 || got.Resolved != 2 || got.FixedItself != 1 || got.Told != 1 {
|
||||
t.Errorf("counts: opened %d resolved %d fixed itself %d told %d", got.Opened, got.Resolved, got.FixedItself, got.Told)
|
||||
}
|
||||
if e := got.Events[3]; !e.At.Equal(*at(0.5)) || e.AckBy != "ana" || e.Told {
|
||||
t.Errorf("B-7 (opened before, closed since) = %+v", e)
|
||||
}
|
||||
if e := got.Events[1]; e.AckBy != "sam" {
|
||||
t.Errorf("B-9 ack = %q", e.AckBy)
|
||||
}
|
||||
if empty := today(nil, nil, since); empty.Events == nil {
|
||||
t.Error("events should be [] not null")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVisited(t *testing.T) {
|
||||
srv := &Server{}
|
||||
t0 := time.Date(2026, 11, 16, 8, 0, 0, 0, time.UTC)
|
||||
if p := srv.visited("ana", t0); !p.IsZero() {
|
||||
t.Errorf("first visit: prev = %s", p)
|
||||
}
|
||||
if p := srv.visited("ana", t0.Add(10*time.Minute)); !p.IsZero() {
|
||||
t.Errorf("same visit: prev = %s", p)
|
||||
}
|
||||
// Away longer than the gap: the visit that ended at +10m is the previous one.
|
||||
if p := srv.visited("ana", t0.Add(5*time.Hour)); !p.Equal(t0.Add(10 * time.Minute)) {
|
||||
t.Errorf("next visit: prev = %s", p)
|
||||
}
|
||||
if p := srv.visited("ana", t0.Add(5*time.Hour+time.Minute)); !p.Equal(t0.Add(10 * time.Minute)) {
|
||||
t.Errorf("polling keeps the previous visit: prev = %s", p)
|
||||
}
|
||||
if p := srv.visited("sam", t0.Add(5*time.Hour)); !p.IsZero() {
|
||||
t.Errorf("another person: prev = %s", p)
|
||||
}
|
||||
if p, l := parseVisit("0 1763280000"); !p.IsZero() || l.Unix() != 1763280000 {
|
||||
t.Errorf("parseVisit = %s %s", p, l)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBusier(t *testing.T) {
|
||||
now := time.Date(2026, 11, 16, 14, 20, 0, 0, time.UTC)
|
||||
usual := map[string]*[24]usualHour{}
|
||||
for _, h := range []string{"calm", "hot", "hungry", "young"} {
|
||||
u := &[24]usualHour{}
|
||||
n := 3
|
||||
if h == "young" {
|
||||
n = 2 // two days of 14:00 aren't enough to say what's usual
|
||||
}
|
||||
u[14] = usualHour{n: n, cpu: 18 * float64(n), mem: 50 * float64(n)}
|
||||
usual[h] = u
|
||||
}
|
||||
got := busier([]HostView{
|
||||
{Name: "calm", CPU: 30, MemPct: 55}, // under twice usual, memory only 5 up
|
||||
{Name: "hot", CPU: 44, MemPct: 52}, // 44 ≥ 2×18 and 26 up
|
||||
{Name: "hungry", CPU: 10, MemPct: 81}, // memory 31 up
|
||||
{Name: "young", CPU: 90, MemPct: 90},
|
||||
{Name: "new", CPU: 90},
|
||||
}, usual, now)
|
||||
var s []string
|
||||
for _, b := range got {
|
||||
s = append(s, b.Host+"/"+b.What)
|
||||
}
|
||||
if j := strings.Join(s, " "); j != "hungry/memory hot/cpu" {
|
||||
t.Errorf("busier = %s", j)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestartedAndFilling(t *testing.T) {
|
||||
hosts := []HostView{{Name: "a", Uptime: 3600}, {Name: "b", Uptime: 90000}, {Name: "c", Uptime: 600}, {Name: "d", Uptime: 60, Silent: true}}
|
||||
got, n := restarted(hosts)
|
||||
if n != 2 || got[0].Host != "c" || got[1].Host != "a" {
|
||||
t.Errorf("restarted = %+v (%d)", got, n)
|
||||
}
|
||||
t0 := time.Date(2026, 11, 9, 0, 0, 0, 0, time.UTC)
|
||||
var hours []state.HostHour
|
||||
for i := range 48 {
|
||||
h := t0.Add(time.Duration(i) * time.Hour)
|
||||
hours = append(hours,
|
||||
state.HostHour{Host: "fills", Hour: h, Disks: []state.DiskSample{{Mount: "/var", Used: uint64(600 + i*2), Total: 1000}}}, // 2 a hour, 306 left: ~6 days
|
||||
state.HostHour{Host: "steady", Hour: h, Disks: []state.DiskSample{{Mount: "/", Used: 900, Total: 1000}}},
|
||||
state.HostHour{Host: "slow", Hour: h, Disks: []state.DiskSample{{Mount: "/", Used: uint64(500 + i/12), Total: 1000}}}, // years
|
||||
)
|
||||
}
|
||||
f := fillingDisks(hours)
|
||||
if len(f) != 1 || f[0].Host != "fills" || f[0].FullDays < 6 || f[0].FullDays > 7 {
|
||||
t.Errorf("filling = %+v", f)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithIncidents(t *testing.T) {
|
||||
attn := []Attention{{Key: "disk/a//", Title: "disk"}, {Key: "", Title: "folded"}, {Key: "unit/b/x", Title: "no incident yet"}}
|
||||
open := []Incident{{ID: "B-3", Key: "disk/a//", Ack: &Ack{By: "ana"}}, {ID: "B-4", Key: ""}}
|
||||
got := withIncidents(attn, open)
|
||||
if got[0].Incident == nil || got[0].Incident.ID != "B-3" || got[0].Incident.Ack.By != "ana" {
|
||||
t.Errorf("disk: %+v", got[0].Incident)
|
||||
}
|
||||
if got[1].Incident != nil || got[2].Incident != nil {
|
||||
t.Errorf("others: %+v %+v", got[1].Incident, got[2].Incident)
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package state
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -62,12 +63,19 @@ func (d *DB) LatestStatuses() (map[string]string, error) {
|
||||
// StatusChanges is every change in [from, to), plus each entity's last
|
||||
// change before from (its status as the period began), oldest first.
|
||||
func (d *DB) StatusChanges(from, to time.Time) ([]Change, error) {
|
||||
return d.StatusChangesOf("", from, to)
|
||||
}
|
||||
|
||||
// StatusChangesOf is StatusChanges for the entities whose names start
|
||||
// with prefix ("svc/": the services).
|
||||
func (d *DB) StatusChangesOf(prefix string, from, to time.Time) ([]Change, error) {
|
||||
like := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(prefix) + "%"
|
||||
rows, err := d.query(`
|
||||
SELECT entity, at, status, `+d.kind.rowOrder()+` AS n FROM status_log s
|
||||
WHERE at < ? AND at = (SELECT MAX(at) FROM status_log WHERE entity = s.entity AND at < ?)
|
||||
WHERE entity LIKE ? ESCAPE '\' AND at < ? AND at = (SELECT MAX(at) FROM status_log WHERE entity = s.entity AND at < ?)
|
||||
UNION ALL
|
||||
SELECT entity, at, status, `+d.kind.rowOrder()+` AS n FROM status_log WHERE at >= ? AND at < ?
|
||||
ORDER BY at, n`, from.Unix(), from.Unix(), from.Unix(), to.Unix())
|
||||
SELECT entity, at, status, `+d.kind.rowOrder()+` AS n FROM status_log WHERE entity LIKE ? ESCAPE '\' AND at >= ? AND at < ?
|
||||
ORDER BY at, n`, like, from.Unix(), from.Unix(), like, from.Unix(), to.Unix())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -48,6 +48,18 @@ func TestStatusHistory(t *testing.T) {
|
||||
if len(got) != 2 {
|
||||
t.Errorf("after prune: %+v", got)
|
||||
}
|
||||
// One kind of entity only; the prefix's _ is a letter, not a wildcard.
|
||||
_ = db.LogStatus("host/mail", "critical", t0.Add(61*24*time.Hour))
|
||||
_ = db.LogStatus("svcXmail", "critical", t0.Add(61*24*time.Hour))
|
||||
_ = db.LogStatus("svc_mail", "critical", t0.Add(61*24*time.Hour))
|
||||
got, _ = db.StatusChangesOf("svc/", t0.Add(61*24*time.Hour), t0.Add(62*24*time.Hour))
|
||||
if len(got) != 2 || got[0].Entity[:4] != "svc/" || got[1].Entity[:4] != "svc/" {
|
||||
t.Errorf("svc/ only: %+v", got)
|
||||
}
|
||||
got, _ = db.StatusChangesOf("svc_", t0, t0.Add(62*24*time.Hour))
|
||||
if len(got) != 1 || got[0].Entity != "svc_mail" {
|
||||
t.Errorf("svc_ only: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func join(s []string) string {
|
||||
|
||||
Reference in new issue
Block a user