Files
cairnobs/internal/server/api.go
T
jcoffey-dev 44ca6f361e
ci / postgres (pull_request) Successful in 3m21s
ci / check (pull_request) Successful in 8m24s
Windows: the event log's errors and criticals on the host's page
A Windows agent reads System and Application every five minutes through
wevtapi: the errors and criticals of the last day, grouped by log, source
and event ID, each with its latest message as Windows formats it. Sent on a
change and hourly; obs-beacond keeps the last and shows it on the host's
page. A critical event in the last hour (an unexpected restart, a failing
disk) is a warning; errors are listed, never alerted on. event_log: false
turns it off.

Run on the Windows Server 2025 VM as the service's own account: both logs
readable, no extra rights.
2026-11-15 02:54:47 -07:00

477 lines
16 KiB
Go

package server
import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"sort"
"strings"
"time"
"github.com/cairnobs/cairnobs/internal/report"
"github.com/cairnobs/cairnobs/internal/units"
"github.com/cairnobs/cairnobs/internal/version"
)
// maxReport bounds an agent's report body. A host with a few hundred
// containers stays well under it.
const maxReport = 4 << 20
func writeJSON(w http.ResponseWriter, code int, v any) {
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Cache-Control", "no-store")
w.WriteHeader(code)
_ = json.NewEncoder(w).Encode(v)
}
func (srv *Server) handleReport(w http.ResponseWriter, r *http.Request) {
agent, ok := srv.agentFor(r)
if !ok {
http.Error(w, srv.agentRefusal(), http.StatusUnauthorized)
return
}
if srv.clusterAgent(agent) {
http.Error(w, agent+" is a cluster agent: it sends workloads, not host reports", http.StatusForbidden)
return
}
var rep report.Report
body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxReport))
if err == nil {
err = json.Unmarshal(body, &rep)
}
if err != nil {
var tooBig *http.MaxBytesError
if errors.As(err, &tooBig) {
http.Error(w, "report too large", http.StatusRequestEntityTooLarge)
return
}
http.Error(w, "bad report: "+err.Error(), http.StatusBadRequest)
return
}
if rep.V != report.Version {
http.Error(w, fmt.Sprintf("protocol version %d, want %d", rep.V, report.Version), http.StatusBadRequest)
return
}
queued, now := r.Header.Get("X-Beacon-Queued") != "", time.Now()
if queued {
// Sent from the agent's queue after obs-beacond was away: history
// for the sparklines, not the host's state now (a newer report
// follows it).
srv.store.History(agent, rep)
} else {
srv.store.Put(agent, rep, now)
srv.forgetSources(agent, rep.Sources)
}
srv.shareReport(agent, body, rep, queued, now)
// Tell the agent which check list it should be running; it fetches the
// list when this differs from what it has.
if _, ver := srv.workFor(agent); ver != "" {
w.Header().Set("X-Beacon-Checks", ver)
}
// What came of actions it took, and whether more wait for it.
srv.actionsAnswer(w, agent, rep.Actions)
w.WriteHeader(http.StatusNoContent)
}
// DiskView is one filesystem as the UI shows it.
type DiskView struct {
Mount string `json:"mount"`
FS string `json:"fs"`
Size uint64 `json:"size"`
Used uint64 `json:"used"`
Pct float64 `json:"pct"`
}
// HostView is one host as the UI shows it.
type HostView struct {
Name string `json:"name"`
Status string `json:"status"` // good, warning, critical, waiting
Word string `json:"word"`
LastSeen *time.Time `json:"last_seen"`
Silent bool `json:"silent"`
// A workstation never pages (obs-beacond.yaml agents: workstation: true).
Workstation bool `json:"workstation,omitempty"`
// VM is the VM this host is, when a cluster's VM has its SMBIOS UUID.
VM *VMLink `json:"vm,omitempty"`
// VMs counts the libvirt VMs this host runs, when it runs libvirt.
VMs *HostVMCount `json:"vms,omitempty"`
Agent string `json:"agent,omitempty"`
Kernel string `json:"kernel,omitempty"`
OS string `json:"os,omitempty"`
Uptime float64 `json:"uptime_s"`
CPUs int `json:"cpus"`
CPU float64 `json:"cpu_pct"`
Load [3]float64 `json:"load"`
MemPct float64 `json:"mem_pct"`
MemTotal uint64 `json:"mem_total"`
SwapPct float64 `json:"swap_pct"`
Disks []DiskView `json:"disks"`
Fullest *DiskView `json:"fullest,omitempty"`
Net report.Net `json:"net"`
History HostHistory `json:"history"`
Containers ContainerSummary `json:"containers"`
ContainersError string `json:"containers_error,omitempty"`
Units []UnitView `json:"units"`
// Events is a Windows host's event log: the errors and criticals of
// the last day (docs/windows.md, step 7).
Events *report.Events `json:"events,omitempty"`
rows []ContainerRow
}
// UnitView is one expected systemd unit as the UI shows it.
type UnitView struct {
report.Unit
Host string `json:"host"`
Severity string `json:"severity"` // good, critical, waiting
Word string `json:"word"`
}
// windowsHost says a report's OS is Windows: its units are Windows
// services (docs/windows.md).
func windowsHost(os string) bool { return strings.HasPrefix(os, "Windows") }
// judgeUnits compares a host's expected units with what its agent saw.
func judgeUnits(host string, want []string, got []report.Unit) []UnitView {
seen := map[string]report.Unit{}
for _, u := range got {
seen[u.Name] = u
}
out := []UnitView{}
for _, n := range want {
n = units.Name(n)
u, ok := seen[n]
v := UnitView{Unit: u, Host: host}
v.Name = n
switch {
case !ok:
v.Severity, v.Word = "waiting", "Not reported yet"
case u.Load == "not-found":
v.Severity, v.Word = "critical", "Not installed"
case u.Active == "active":
v.Severity, v.Word = "good", "Running"
if u.Sub != "running" && u.Sub != "" {
v.Word = "Active (" + u.Sub + ")"
}
case u.Active == "activating" || u.Active == "reloading":
v.Severity, v.Word = "warning", "Starting"
default:
v.Severity, v.Word = "critical", strings.ToUpper(u.Active[:1])+u.Active[1:]
}
out = append(out, v)
}
return out
}
// HostHistory is the in-memory history for sparklines, oldest first.
type HostHistory struct {
CPU []float64 `json:"cpu"`
Mem []float64 `json:"mem"`
Rx []float64 `json:"rx"`
Tx []float64 `json:"tx"`
}
// Attention is one "needs attention" card.
type Attention struct {
Severity string `json:"severity"`
Title string `json:"title"`
Detail string `json:"detail"`
Href string `json:"href"`
Host string `json:"host"`
}
func nonNil(s []float64) []float64 {
if s == nil {
return []float64{}
}
return s
}
func pct(used, total uint64) float64 {
if total == 0 {
return 0
}
return 100 * float64(used) / float64(total)
}
func plural(n int, one, many string) string {
if n == 1 {
return fmt.Sprintf("%d %s", n, one)
}
return fmt.Sprintf("%d %s", n, many)
}
// Condition is one thing wrong right now. Key is stable for as long as the
// problem lasts (the same container on the same host, the same disk), so
// the alert engine can follow it from one evaluation to the next.
type Condition struct {
Key string `json:"key"`
Severity string `json:"severity"` // warning or critical
Title string `json:"title"`
Detail string `json:"detail"`
Host string `json:"host"`
Href string `json:"href"`
Affected string `json:"affected,omitempty"` // what else this takes down (root-cause grouping)
Grouped []GroupedBy `json:"grouped,omitempty"` // the learned dependencies that grouped it, when any did
Map string `json:"map,omitempty"` // where it is on the maps: a page path with its focus
}
// view judges every host now: what the UI shows, and the conditions the
// alert engine works from.
func (srv *Server) view(now time.Time) ([]HostView, []Condition) {
// Empty lists, never nil: the browser gets [] rather than null.
hosts := []HostView{}
conds := []Condition{}
expected := srv.currentExpected()
for _, h := range srv.store.Snapshot() {
v := HostView{Name: h.name, Disks: []DiskView{}, Units: []UnitView{}, History: HostHistory{CPU: []float64{}, Mem: []float64{}, Rx: []float64{}, Tx: []float64{}}}
href := "/hosts/" + h.name
ws := srv.workstation(h.name)
v.Workstation = ws
// A workstation's problems never page: warning is as high as they go.
capped := func(sev string) string {
if ws {
return "warning"
}
return sev
}
if h.seen.IsZero() {
v.Status, v.Word = "waiting", "Waiting for its first report"
// A host that never reports after obs-beacond starts is as bad as one that went quiet.
if since := now.Sub(srv.started); since > srv.cfg.StaleAfter {
v.Status, v.Word, v.Silent = capped("critical"), "Silent", true
conds = append(conds, Condition{Key: "silent/" + h.name, Severity: capped("critical"), Title: h.name + " has gone silent",
Detail: "No report since obs-beacond started " + since.Round(time.Second).String() + " ago. Its agent, the host or the link to it is down.", Host: h.name, Href: href})
}
hosts = append(hosts, v)
continue
}
seen := h.seen
r := h.last
v.LastSeen = &seen
if r.VMs != nil {
n := &HostVMCount{Total: len(r.VMs.List), Error: r.VMs.Error, Href: workloadHref(h.name, "", "")}
for _, w := range r.VMs.List {
if vmRunning(w.State) {
n.Running++
}
}
v.VMs = n
}
v.Agent, v.Kernel, v.OS, v.Uptime, v.CPUs, v.CPU, v.Load = r.Agent, r.Kernel, r.OS, r.Uptime, r.CPUs, r.CPU, r.Load
v.MemPct, v.MemTotal = memPct(r.Mem), r.Mem.Total
v.SwapPct = pct(r.Mem.SwapTotal-min(r.Mem.SwapFree, r.Mem.SwapTotal), r.Mem.SwapTotal)
v.Net = r.Net
v.History = HostHistory{CPU: nonNil(h.cpu), Mem: nonNil(h.mem), Rx: nonNil(h.rx), Tx: nonNil(h.tx)}
for _, d := range r.Disks {
dv := DiskView{Mount: d.Mount, FS: d.FS, Size: d.Size, Used: d.Used, Pct: pct(d.Used, d.Used+d.Avail)}
v.Disks = append(v.Disks, dv)
if v.Fullest == nil || dv.Pct > v.Fullest.Pct {
f := dv
v.Fullest = &f
}
}
// A workstation's containers come and go with whatever is being worked
// on: they aren't judged, drawn on the maps or compared for drift.
if r.Containers != nil && !ws {
v.ContainersError = r.Containers.Error
v.rows, v.Containers = expected.Judge(h.name, r.Containers.List)
}
crit, warn := 0, 0
add := func(c Condition) {
c.Host = h.name
c.Severity = capped(c.Severity)
if c.Severity == "critical" {
crit++
} else {
warn++
}
conds = append(conds, c)
}
if age := now.Sub(h.seen); age > srv.cfg.StaleAfter {
v.Silent = true
add(Condition{Key: "silent/" + h.name, Severity: "critical", Title: h.name + " has gone silent",
Detail: "No report for " + age.Round(time.Second).String() + ". Its agent, the host or the link to it is down.", Href: href})
}
for _, d := range v.Disks {
key := "disk/" + h.name + "/" + d.Mount
switch {
case d.Pct >= 90:
add(Condition{Key: key, Severity: "critical", Title: fmt.Sprintf("%s %s is %.0f%% full", h.name, d.Mount, d.Pct), Detail: "Alert at 90%.", Host: "", Href: href})
case d.Pct >= 80:
add(Condition{Key: key, Severity: "warning", Title: fmt.Sprintf("%s %s is %.0f%% full", h.name, d.Mount, d.Pct), Detail: "Warning at 80%.", Host: "", Href: href})
}
}
if r.Mem.Total > 0 && float64(r.Mem.Available) < 0.1*float64(r.Mem.Total) {
add(Condition{Key: "memory/" + h.name, Severity: "warning", Title: h.name + " is low on memory", Detail: fmt.Sprintf("%.0f%% in use.", v.MemPct), Host: "", Href: href})
}
for _, row := range v.rows {
if row.Verdict.Severity != "critical" && row.Verdict.Severity != "warning" {
continue
}
add(Condition{Key: "container/" + h.name + "/" + row.Name, Severity: row.Verdict.Severity,
Title: fmt.Sprintf("%s on %s: %s", row.Name, h.name, strings.ToLower(row.Verdict.Word)), Detail: row.Verdict.Detail, Host: "", Href: "/containers"})
}
v.Units = judgeUnits(h.name, expected.Hosts[h.name].Units, r.Units)
manager, win := "systemd", windowsHost(r.OS)
if win {
manager = "Windows"
}
for _, u := range v.Units {
if u.Severity == "critical" {
name := u.Name
if win {
name = units.ServiceName(name)
}
add(Condition{Key: "unit/" + h.name + "/" + u.Name, Severity: "critical", Title: fmt.Sprintf("%s on %s: %s", name, h.name, strings.ToLower(u.Word)),
Detail: manager + " reports it " + u.Active + " (" + u.Sub + ").", Host: "", Href: href})
}
}
if h.events != nil && now.Sub(h.events.At) < 2*time.Hour {
v.Events = h.events
for _, c := range eventConditions(h.name, h.events, now) {
add(c)
}
}
if v.ContainersError != "" {
add(Condition{Key: "containers-unreadable/" + h.name, Severity: "warning", Title: "Can't read containers on " + h.name, Detail: v.ContainersError, Host: "", Href: href})
}
switch {
case v.Silent:
v.Status, v.Word = capped("critical"), "Silent"
case crit > 0:
v.Status, v.Word = "critical", plural(crit, "critical", "critical")
case warn > 0:
v.Status, v.Word = "warning", plural(warn, "warning", "warnings")
default:
v.Status, v.Word = "good", "Up"
}
hosts = append(hosts, v)
}
conds = append(conds, srv.cloudflareConditions(now)...)
conds = append(conds, srv.akamaiConditions(now)...)
conds = append(conds, srv.mailConditions(now)...)
conds = append(conds, srv.backupConditions(now)...)
conds = append(conds, srv.imageDrift(hosts, now)...)
_, checkConds := srv.judgeChecks(now)
conds = append(conds, checkConds...)
_, dlConds := srv.dataLayerView(now)
conds = append(conds, dlConds...)
conds = append(conds, srv.workloadConditions(now)...)
conds = append(conds, srv.certConditions(now)...)
if byHost, _ := srv.vmLinks(); len(byHost) > 0 {
explainSilent(conds, byHost)
for i := range hosts {
if v, ok := byHost[hosts[i].Name]; ok {
hosts[i].VM = &v
}
}
}
sort.SliceStable(conds, func(i, j int) bool { return conds[i].Severity == "critical" && conds[j].Severity != "critical" })
return hosts, conds
}
// attention turns conditions into the dashboard's cards: per host, more
// than four container problems fold into one "N more" card.
func attention(conds []Condition) []Attention {
out := []Attention{}
shown := map[string]int{}
total := map[string]int{}
for _, c := range conds {
if strings.HasPrefix(c.Key, "container/") {
total[c.Host]++
}
}
folded := map[string]bool{}
for _, c := range conds {
if strings.HasPrefix(c.Key, "container/") && total[c.Host] > 4 {
if shown[c.Host] == 3 {
if !folded[c.Host] {
folded[c.Host] = true
out = append(out, Attention{"warning", fmt.Sprintf("%d more containers on %s need a look", total[c.Host]-3, c.Host), "", "/containers", c.Host})
}
continue
}
shown[c.Host]++
}
out = append(out, Attention{c.Severity, c.Title, c.Detail, c.Href, c.Host})
}
return out
}
func (srv *Server) handleOverview(w http.ResponseWriter, r *http.Request) {
now := time.Now()
hosts, conds := srv.view(now)
attn := attention(srv.groupConditions(conds, srv.alerts.firing))
var sum ContainerSummary
for _, h := range hosts {
sum.add(h.Containers)
}
open, _ := srv.alerts.snapshot()
firing, unacked := 0, 0
for _, inc := range open {
if inc.Fired != nil {
firing++
if inc.Ack == nil {
unacked++
}
}
}
checks := map[string]any{"down": 0, "severity": "warning"}
views, _ := srv.judgeChecks(now)
down := 0
for _, v := range views {
if v.Word == "Down" {
down++
if v.Status == "critical" {
checks["severity"] = "critical"
}
}
}
checks["down"] = down
writeJSON(w, http.StatusOK, map[string]any{
// hosts is all of them in a small fleet, else the most in need
// (lists.go); fleet counts every one.
"now": now, "version": version.Version, "hosts": overviewList(hosts), "fleet": summarize(hosts), "checks": checks,
"attention": attn, "containers": sum, "clusters": srv.clusterViews(now, conds),
"alerts": map[string]int{"firing": firing, "unacknowledged": unacked},
"features": srv.features(),
})
}
func (srv *Server) handleHost(w http.ResponseWriter, r *http.Request) {
name := r.PathValue("name")
hosts, conds := srv.view(time.Now())
attn := attention(conds)
for _, h := range hosts {
if h.Name == name {
mine := []Attention{}
for _, a := range attn {
if a.Host == name {
mine = append(mine, a)
}
}
rows := h.rows
if rows == nil {
rows = []ContainerRow{}
}
writeJSON(w, http.StatusOK, map[string]any{"host": h, "containers": srv.logsFor(r, rows), "attention": mine})
return
}
}
writeJSON(w, http.StatusNotFound, map[string]string{"error": "no such host"})
}
// HostVMCount is how many libvirt VMs a host runs, and where to see them.
type HostVMCount struct {
Total int `json:"total"`
Running int `json:"running"`
Error string `json:"error,omitempty"`
Href string `json:"href"`
}