A Windows agent reads System and Application every five minutes through wevtapi: the errors and criticals of the last day, grouped by log, source and event ID, each with its latest message as Windows formats it. Sent on a change and hourly; obs-beacond keeps the last and shows it on the host's page. A critical event in the last hour (an unexpected restart, a failing disk) is a warning; errors are listed, never alerted on. event_log: false turns it off. Run on the Windows Server 2025 VM as the service's own account: both logs readable, no extra rights.
477 lines
16 KiB
Go
477 lines
16 KiB
Go
package server
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/cairnobs/cairnobs/internal/report"
|
|
"github.com/cairnobs/cairnobs/internal/units"
|
|
"github.com/cairnobs/cairnobs/internal/version"
|
|
)
|
|
|
|
// maxReport bounds an agent's report body. A host with a few hundred
|
|
// containers stays well under it.
|
|
const maxReport = 4 << 20
|
|
|
|
func writeJSON(w http.ResponseWriter, code int, v any) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.Header().Set("Cache-Control", "no-store")
|
|
w.WriteHeader(code)
|
|
_ = json.NewEncoder(w).Encode(v)
|
|
}
|
|
|
|
func (srv *Server) handleReport(w http.ResponseWriter, r *http.Request) {
|
|
agent, ok := srv.agentFor(r)
|
|
if !ok {
|
|
http.Error(w, srv.agentRefusal(), http.StatusUnauthorized)
|
|
return
|
|
}
|
|
if srv.clusterAgent(agent) {
|
|
http.Error(w, agent+" is a cluster agent: it sends workloads, not host reports", http.StatusForbidden)
|
|
return
|
|
}
|
|
var rep report.Report
|
|
body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxReport))
|
|
if err == nil {
|
|
err = json.Unmarshal(body, &rep)
|
|
}
|
|
if err != nil {
|
|
var tooBig *http.MaxBytesError
|
|
if errors.As(err, &tooBig) {
|
|
http.Error(w, "report too large", http.StatusRequestEntityTooLarge)
|
|
return
|
|
}
|
|
http.Error(w, "bad report: "+err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
if rep.V != report.Version {
|
|
http.Error(w, fmt.Sprintf("protocol version %d, want %d", rep.V, report.Version), http.StatusBadRequest)
|
|
return
|
|
}
|
|
queued, now := r.Header.Get("X-Beacon-Queued") != "", time.Now()
|
|
if queued {
|
|
// Sent from the agent's queue after obs-beacond was away: history
|
|
// for the sparklines, not the host's state now (a newer report
|
|
// follows it).
|
|
srv.store.History(agent, rep)
|
|
} else {
|
|
srv.store.Put(agent, rep, now)
|
|
srv.forgetSources(agent, rep.Sources)
|
|
}
|
|
srv.shareReport(agent, body, rep, queued, now)
|
|
// Tell the agent which check list it should be running; it fetches the
|
|
// list when this differs from what it has.
|
|
if _, ver := srv.workFor(agent); ver != "" {
|
|
w.Header().Set("X-Beacon-Checks", ver)
|
|
}
|
|
// What came of actions it took, and whether more wait for it.
|
|
srv.actionsAnswer(w, agent, rep.Actions)
|
|
w.WriteHeader(http.StatusNoContent)
|
|
}
|
|
|
|
// DiskView is one filesystem as the UI shows it.
|
|
type DiskView struct {
|
|
Mount string `json:"mount"`
|
|
FS string `json:"fs"`
|
|
Size uint64 `json:"size"`
|
|
Used uint64 `json:"used"`
|
|
Pct float64 `json:"pct"`
|
|
}
|
|
|
|
// HostView is one host as the UI shows it.
|
|
type HostView struct {
|
|
Name string `json:"name"`
|
|
Status string `json:"status"` // good, warning, critical, waiting
|
|
Word string `json:"word"`
|
|
LastSeen *time.Time `json:"last_seen"`
|
|
Silent bool `json:"silent"`
|
|
// A workstation never pages (obs-beacond.yaml agents: workstation: true).
|
|
Workstation bool `json:"workstation,omitempty"`
|
|
// VM is the VM this host is, when a cluster's VM has its SMBIOS UUID.
|
|
VM *VMLink `json:"vm,omitempty"`
|
|
// VMs counts the libvirt VMs this host runs, when it runs libvirt.
|
|
VMs *HostVMCount `json:"vms,omitempty"`
|
|
|
|
Agent string `json:"agent,omitempty"`
|
|
Kernel string `json:"kernel,omitempty"`
|
|
OS string `json:"os,omitempty"`
|
|
Uptime float64 `json:"uptime_s"`
|
|
CPUs int `json:"cpus"`
|
|
CPU float64 `json:"cpu_pct"`
|
|
Load [3]float64 `json:"load"`
|
|
MemPct float64 `json:"mem_pct"`
|
|
MemTotal uint64 `json:"mem_total"`
|
|
SwapPct float64 `json:"swap_pct"`
|
|
Disks []DiskView `json:"disks"`
|
|
Fullest *DiskView `json:"fullest,omitempty"`
|
|
Net report.Net `json:"net"`
|
|
History HostHistory `json:"history"`
|
|
|
|
Containers ContainerSummary `json:"containers"`
|
|
ContainersError string `json:"containers_error,omitempty"`
|
|
Units []UnitView `json:"units"`
|
|
// Events is a Windows host's event log: the errors and criticals of
|
|
// the last day (docs/windows.md, step 7).
|
|
Events *report.Events `json:"events,omitempty"`
|
|
rows []ContainerRow
|
|
}
|
|
|
|
// UnitView is one expected systemd unit as the UI shows it.
|
|
type UnitView struct {
|
|
report.Unit
|
|
Host string `json:"host"`
|
|
Severity string `json:"severity"` // good, critical, waiting
|
|
Word string `json:"word"`
|
|
}
|
|
|
|
// windowsHost says a report's OS is Windows: its units are Windows
|
|
// services (docs/windows.md).
|
|
func windowsHost(os string) bool { return strings.HasPrefix(os, "Windows") }
|
|
|
|
// judgeUnits compares a host's expected units with what its agent saw.
|
|
func judgeUnits(host string, want []string, got []report.Unit) []UnitView {
|
|
seen := map[string]report.Unit{}
|
|
for _, u := range got {
|
|
seen[u.Name] = u
|
|
}
|
|
out := []UnitView{}
|
|
for _, n := range want {
|
|
n = units.Name(n)
|
|
u, ok := seen[n]
|
|
v := UnitView{Unit: u, Host: host}
|
|
v.Name = n
|
|
switch {
|
|
case !ok:
|
|
v.Severity, v.Word = "waiting", "Not reported yet"
|
|
case u.Load == "not-found":
|
|
v.Severity, v.Word = "critical", "Not installed"
|
|
case u.Active == "active":
|
|
v.Severity, v.Word = "good", "Running"
|
|
if u.Sub != "running" && u.Sub != "" {
|
|
v.Word = "Active (" + u.Sub + ")"
|
|
}
|
|
case u.Active == "activating" || u.Active == "reloading":
|
|
v.Severity, v.Word = "warning", "Starting"
|
|
default:
|
|
v.Severity, v.Word = "critical", strings.ToUpper(u.Active[:1])+u.Active[1:]
|
|
}
|
|
out = append(out, v)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// HostHistory is the in-memory history for sparklines, oldest first.
|
|
type HostHistory struct {
|
|
CPU []float64 `json:"cpu"`
|
|
Mem []float64 `json:"mem"`
|
|
Rx []float64 `json:"rx"`
|
|
Tx []float64 `json:"tx"`
|
|
}
|
|
|
|
// Attention is one "needs attention" card.
|
|
type Attention struct {
|
|
Severity string `json:"severity"`
|
|
Title string `json:"title"`
|
|
Detail string `json:"detail"`
|
|
Href string `json:"href"`
|
|
Host string `json:"host"`
|
|
}
|
|
|
|
func nonNil(s []float64) []float64 {
|
|
if s == nil {
|
|
return []float64{}
|
|
}
|
|
return s
|
|
}
|
|
|
|
func pct(used, total uint64) float64 {
|
|
if total == 0 {
|
|
return 0
|
|
}
|
|
return 100 * float64(used) / float64(total)
|
|
}
|
|
|
|
func plural(n int, one, many string) string {
|
|
if n == 1 {
|
|
return fmt.Sprintf("%d %s", n, one)
|
|
}
|
|
return fmt.Sprintf("%d %s", n, many)
|
|
}
|
|
|
|
// Condition is one thing wrong right now. Key is stable for as long as the
|
|
// problem lasts (the same container on the same host, the same disk), so
|
|
// the alert engine can follow it from one evaluation to the next.
|
|
type Condition struct {
|
|
Key string `json:"key"`
|
|
Severity string `json:"severity"` // warning or critical
|
|
Title string `json:"title"`
|
|
Detail string `json:"detail"`
|
|
Host string `json:"host"`
|
|
Href string `json:"href"`
|
|
Affected string `json:"affected,omitempty"` // what else this takes down (root-cause grouping)
|
|
Grouped []GroupedBy `json:"grouped,omitempty"` // the learned dependencies that grouped it, when any did
|
|
Map string `json:"map,omitempty"` // where it is on the maps: a page path with its focus
|
|
}
|
|
|
|
// view judges every host now: what the UI shows, and the conditions the
|
|
// alert engine works from.
|
|
func (srv *Server) view(now time.Time) ([]HostView, []Condition) {
|
|
// Empty lists, never nil: the browser gets [] rather than null.
|
|
hosts := []HostView{}
|
|
conds := []Condition{}
|
|
expected := srv.currentExpected()
|
|
for _, h := range srv.store.Snapshot() {
|
|
v := HostView{Name: h.name, Disks: []DiskView{}, Units: []UnitView{}, History: HostHistory{CPU: []float64{}, Mem: []float64{}, Rx: []float64{}, Tx: []float64{}}}
|
|
href := "/hosts/" + h.name
|
|
ws := srv.workstation(h.name)
|
|
v.Workstation = ws
|
|
// A workstation's problems never page: warning is as high as they go.
|
|
capped := func(sev string) string {
|
|
if ws {
|
|
return "warning"
|
|
}
|
|
return sev
|
|
}
|
|
if h.seen.IsZero() {
|
|
v.Status, v.Word = "waiting", "Waiting for its first report"
|
|
// A host that never reports after obs-beacond starts is as bad as one that went quiet.
|
|
if since := now.Sub(srv.started); since > srv.cfg.StaleAfter {
|
|
v.Status, v.Word, v.Silent = capped("critical"), "Silent", true
|
|
conds = append(conds, Condition{Key: "silent/" + h.name, Severity: capped("critical"), Title: h.name + " has gone silent",
|
|
Detail: "No report since obs-beacond started " + since.Round(time.Second).String() + " ago. Its agent, the host or the link to it is down.", Host: h.name, Href: href})
|
|
}
|
|
hosts = append(hosts, v)
|
|
continue
|
|
}
|
|
seen := h.seen
|
|
r := h.last
|
|
v.LastSeen = &seen
|
|
if r.VMs != nil {
|
|
n := &HostVMCount{Total: len(r.VMs.List), Error: r.VMs.Error, Href: workloadHref(h.name, "", "")}
|
|
for _, w := range r.VMs.List {
|
|
if vmRunning(w.State) {
|
|
n.Running++
|
|
}
|
|
}
|
|
v.VMs = n
|
|
}
|
|
v.Agent, v.Kernel, v.OS, v.Uptime, v.CPUs, v.CPU, v.Load = r.Agent, r.Kernel, r.OS, r.Uptime, r.CPUs, r.CPU, r.Load
|
|
v.MemPct, v.MemTotal = memPct(r.Mem), r.Mem.Total
|
|
v.SwapPct = pct(r.Mem.SwapTotal-min(r.Mem.SwapFree, r.Mem.SwapTotal), r.Mem.SwapTotal)
|
|
v.Net = r.Net
|
|
v.History = HostHistory{CPU: nonNil(h.cpu), Mem: nonNil(h.mem), Rx: nonNil(h.rx), Tx: nonNil(h.tx)}
|
|
for _, d := range r.Disks {
|
|
dv := DiskView{Mount: d.Mount, FS: d.FS, Size: d.Size, Used: d.Used, Pct: pct(d.Used, d.Used+d.Avail)}
|
|
v.Disks = append(v.Disks, dv)
|
|
if v.Fullest == nil || dv.Pct > v.Fullest.Pct {
|
|
f := dv
|
|
v.Fullest = &f
|
|
}
|
|
}
|
|
// A workstation's containers come and go with whatever is being worked
|
|
// on: they aren't judged, drawn on the maps or compared for drift.
|
|
if r.Containers != nil && !ws {
|
|
v.ContainersError = r.Containers.Error
|
|
v.rows, v.Containers = expected.Judge(h.name, r.Containers.List)
|
|
}
|
|
|
|
crit, warn := 0, 0
|
|
add := func(c Condition) {
|
|
c.Host = h.name
|
|
c.Severity = capped(c.Severity)
|
|
if c.Severity == "critical" {
|
|
crit++
|
|
} else {
|
|
warn++
|
|
}
|
|
conds = append(conds, c)
|
|
}
|
|
if age := now.Sub(h.seen); age > srv.cfg.StaleAfter {
|
|
v.Silent = true
|
|
add(Condition{Key: "silent/" + h.name, Severity: "critical", Title: h.name + " has gone silent",
|
|
Detail: "No report for " + age.Round(time.Second).String() + ". Its agent, the host or the link to it is down.", Href: href})
|
|
}
|
|
for _, d := range v.Disks {
|
|
key := "disk/" + h.name + "/" + d.Mount
|
|
switch {
|
|
case d.Pct >= 90:
|
|
add(Condition{Key: key, Severity: "critical", Title: fmt.Sprintf("%s %s is %.0f%% full", h.name, d.Mount, d.Pct), Detail: "Alert at 90%.", Host: "", Href: href})
|
|
case d.Pct >= 80:
|
|
add(Condition{Key: key, Severity: "warning", Title: fmt.Sprintf("%s %s is %.0f%% full", h.name, d.Mount, d.Pct), Detail: "Warning at 80%.", Host: "", Href: href})
|
|
}
|
|
}
|
|
if r.Mem.Total > 0 && float64(r.Mem.Available) < 0.1*float64(r.Mem.Total) {
|
|
add(Condition{Key: "memory/" + h.name, Severity: "warning", Title: h.name + " is low on memory", Detail: fmt.Sprintf("%.0f%% in use.", v.MemPct), Host: "", Href: href})
|
|
}
|
|
for _, row := range v.rows {
|
|
if row.Verdict.Severity != "critical" && row.Verdict.Severity != "warning" {
|
|
continue
|
|
}
|
|
add(Condition{Key: "container/" + h.name + "/" + row.Name, Severity: row.Verdict.Severity,
|
|
Title: fmt.Sprintf("%s on %s: %s", row.Name, h.name, strings.ToLower(row.Verdict.Word)), Detail: row.Verdict.Detail, Host: "", Href: "/containers"})
|
|
}
|
|
v.Units = judgeUnits(h.name, expected.Hosts[h.name].Units, r.Units)
|
|
manager, win := "systemd", windowsHost(r.OS)
|
|
if win {
|
|
manager = "Windows"
|
|
}
|
|
for _, u := range v.Units {
|
|
if u.Severity == "critical" {
|
|
name := u.Name
|
|
if win {
|
|
name = units.ServiceName(name)
|
|
}
|
|
add(Condition{Key: "unit/" + h.name + "/" + u.Name, Severity: "critical", Title: fmt.Sprintf("%s on %s: %s", name, h.name, strings.ToLower(u.Word)),
|
|
Detail: manager + " reports it " + u.Active + " (" + u.Sub + ").", Host: "", Href: href})
|
|
}
|
|
}
|
|
if h.events != nil && now.Sub(h.events.At) < 2*time.Hour {
|
|
v.Events = h.events
|
|
for _, c := range eventConditions(h.name, h.events, now) {
|
|
add(c)
|
|
}
|
|
}
|
|
if v.ContainersError != "" {
|
|
add(Condition{Key: "containers-unreadable/" + h.name, Severity: "warning", Title: "Can't read containers on " + h.name, Detail: v.ContainersError, Host: "", Href: href})
|
|
}
|
|
|
|
switch {
|
|
case v.Silent:
|
|
v.Status, v.Word = capped("critical"), "Silent"
|
|
case crit > 0:
|
|
v.Status, v.Word = "critical", plural(crit, "critical", "critical")
|
|
case warn > 0:
|
|
v.Status, v.Word = "warning", plural(warn, "warning", "warnings")
|
|
default:
|
|
v.Status, v.Word = "good", "Up"
|
|
}
|
|
hosts = append(hosts, v)
|
|
}
|
|
conds = append(conds, srv.cloudflareConditions(now)...)
|
|
conds = append(conds, srv.akamaiConditions(now)...)
|
|
conds = append(conds, srv.mailConditions(now)...)
|
|
conds = append(conds, srv.backupConditions(now)...)
|
|
conds = append(conds, srv.imageDrift(hosts, now)...)
|
|
_, checkConds := srv.judgeChecks(now)
|
|
conds = append(conds, checkConds...)
|
|
_, dlConds := srv.dataLayerView(now)
|
|
conds = append(conds, dlConds...)
|
|
conds = append(conds, srv.workloadConditions(now)...)
|
|
conds = append(conds, srv.certConditions(now)...)
|
|
if byHost, _ := srv.vmLinks(); len(byHost) > 0 {
|
|
explainSilent(conds, byHost)
|
|
for i := range hosts {
|
|
if v, ok := byHost[hosts[i].Name]; ok {
|
|
hosts[i].VM = &v
|
|
}
|
|
}
|
|
}
|
|
sort.SliceStable(conds, func(i, j int) bool { return conds[i].Severity == "critical" && conds[j].Severity != "critical" })
|
|
return hosts, conds
|
|
}
|
|
|
|
// attention turns conditions into the dashboard's cards: per host, more
|
|
// than four container problems fold into one "N more" card.
|
|
func attention(conds []Condition) []Attention {
|
|
out := []Attention{}
|
|
shown := map[string]int{}
|
|
total := map[string]int{}
|
|
for _, c := range conds {
|
|
if strings.HasPrefix(c.Key, "container/") {
|
|
total[c.Host]++
|
|
}
|
|
}
|
|
folded := map[string]bool{}
|
|
for _, c := range conds {
|
|
if strings.HasPrefix(c.Key, "container/") && total[c.Host] > 4 {
|
|
if shown[c.Host] == 3 {
|
|
if !folded[c.Host] {
|
|
folded[c.Host] = true
|
|
out = append(out, Attention{"warning", fmt.Sprintf("%d more containers on %s need a look", total[c.Host]-3, c.Host), "", "/containers", c.Host})
|
|
}
|
|
continue
|
|
}
|
|
shown[c.Host]++
|
|
}
|
|
out = append(out, Attention{c.Severity, c.Title, c.Detail, c.Href, c.Host})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (srv *Server) handleOverview(w http.ResponseWriter, r *http.Request) {
|
|
now := time.Now()
|
|
hosts, conds := srv.view(now)
|
|
attn := attention(srv.groupConditions(conds, srv.alerts.firing))
|
|
var sum ContainerSummary
|
|
for _, h := range hosts {
|
|
sum.add(h.Containers)
|
|
}
|
|
open, _ := srv.alerts.snapshot()
|
|
firing, unacked := 0, 0
|
|
for _, inc := range open {
|
|
if inc.Fired != nil {
|
|
firing++
|
|
if inc.Ack == nil {
|
|
unacked++
|
|
}
|
|
}
|
|
}
|
|
checks := map[string]any{"down": 0, "severity": "warning"}
|
|
views, _ := srv.judgeChecks(now)
|
|
down := 0
|
|
for _, v := range views {
|
|
if v.Word == "Down" {
|
|
down++
|
|
if v.Status == "critical" {
|
|
checks["severity"] = "critical"
|
|
}
|
|
}
|
|
}
|
|
checks["down"] = down
|
|
writeJSON(w, http.StatusOK, map[string]any{
|
|
// hosts is all of them in a small fleet, else the most in need
|
|
// (lists.go); fleet counts every one.
|
|
"now": now, "version": version.Version, "hosts": overviewList(hosts), "fleet": summarize(hosts), "checks": checks,
|
|
"attention": attn, "containers": sum, "clusters": srv.clusterViews(now, conds),
|
|
"alerts": map[string]int{"firing": firing, "unacknowledged": unacked},
|
|
"features": srv.features(),
|
|
})
|
|
}
|
|
|
|
func (srv *Server) handleHost(w http.ResponseWriter, r *http.Request) {
|
|
name := r.PathValue("name")
|
|
hosts, conds := srv.view(time.Now())
|
|
attn := attention(conds)
|
|
for _, h := range hosts {
|
|
if h.Name == name {
|
|
mine := []Attention{}
|
|
for _, a := range attn {
|
|
if a.Host == name {
|
|
mine = append(mine, a)
|
|
}
|
|
}
|
|
rows := h.rows
|
|
if rows == nil {
|
|
rows = []ContainerRow{}
|
|
}
|
|
writeJSON(w, http.StatusOK, map[string]any{"host": h, "containers": srv.logsFor(r, rows), "attention": mine})
|
|
return
|
|
}
|
|
}
|
|
writeJSON(w, http.StatusNotFound, map[string]string{"error": "no such host"})
|
|
}
|
|
|
|
// HostVMCount is how many libvirt VMs a host runs, and where to see them.
|
|
type HostVMCount struct {
|
|
Total int `json:"total"`
|
|
Running int `json:"running"`
|
|
Error string `json:"error,omitempty"`
|
|
Href string `json:"href"`
|
|
}
|