Files
cairnobs/internal/server/checks.go
T
jcoffey-dev db156c4f06
release / release (push) Successful in 7m6s
Cairn OBS
The new Cairn OBS: a self-hosted monitor for networks, hosts and
services. Licensed AGPLv3.
2026-11-14 03:27:09 -07:00

290 lines
8.5 KiB
Go

package server
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"net/http"
"os"
"slices"
"sort"
"strings"
"sync"
"time"
"gopkg.in/yaml.v3"
"github.com/cairnobs/cairnobs/internal/check"
"github.com/cairnobs/cairnobs/internal/fetch"
"github.com/cairnobs/cairnobs/internal/report"
"github.com/cairnobs/cairnobs/internal/units"
)
// Checks is checks.yaml: the active checks agents run.
type Checks struct {
Checks []check.Spec `yaml:"checks"`
}
// LoadChecks reads and checks a checks file.
func LoadChecks(p string, agents []AgentConfig) (*Checks, error) {
b, err := os.ReadFile(p)
if err != nil {
return nil, err
}
c, err := parseChecks(b, agentNames(agents))
if err != nil {
return nil, fmt.Errorf("%s: %w", p, err)
}
return c, nil
}
// parseChecks reads a checks document and checks it. Every host in a
// check's `from` must be in known; nil known skips that.
func parseChecks(b []byte, known map[string]bool) (*Checks, error) {
var c Checks
dec := yaml.NewDecoder(bytes.NewReader(b))
dec.KnownFields(true)
if err := dec.Decode(&c); err != nil && !errors.Is(err, io.EOF) {
return nil, err
}
if err := c.check(known); err != nil {
return nil, err
}
return &c, nil
}
// check fills in each check's defaults and refuses a list that can't run.
func (c *Checks) check(known map[string]bool) error {
ids := map[string]bool{}
for i := range c.Checks {
s := &c.Checks[i]
if err := s.Check(); err != nil {
return err
}
if ids[s.ID] {
return fmt.Errorf("two checks called %q", s.ID)
}
ids[s.ID] = true
if known == nil {
continue
}
for _, h := range s.From {
if !known[h] {
return fmt.Errorf("check %s: no host called %q", s.ID, h)
}
}
}
return nil
}
func agentNames(agents []AgentConfig) map[string]bool {
known := map[string]bool{}
for _, a := range agents {
known[a.Name] = true
}
return known
}
// checksFile keeps checks.yaml current: re-read when it changes on disk; a
// file that doesn't parse is logged and the last good list stays. Without
// a file, the checks are edited in the UI and kept in the database (raw is
// what was entered, before defaults; settings.go).
type checksFile struct {
mu sync.Mutex
path string
stat os.FileInfo
cur *Checks
raw []check.Spec
}
func (srv *Server) currentChecks() *Checks {
f := &srv.checks
f.mu.Lock()
defer f.mu.Unlock()
if f.path == "" {
return f.cur
}
st, err := os.Stat(f.path)
if err != nil || (f.stat != nil && st.ModTime().Equal(f.stat.ModTime()) && st.Size() == f.stat.Size()) {
return f.cur
}
first := f.stat == nil
f.stat = st
c, err := LoadChecks(f.path, srv.agentList())
if err != nil {
log.Printf("checks: %v; keeping the last good list", err)
return f.cur
}
f.cur = c
if !first {
srv.audit("obs-beacond", "reloaded the checks", fmt.Sprintf("%s (%d checks)", f.path, len(c.Checks)))
}
return c
}
// agentWork is what an agent is asked to do beyond its own report: the
// checks it runs and the systemd units it watches.
type agentWork struct {
Checks []check.Spec `json:"checks"`
Units []string `json:"units"`
Fetches []fetch.Spec `json:"fetches"`
}
// workFor is an agent's work and its version (empty: nothing to do).
func (srv *Server) workFor(agent string) (agentWork, string) {
w := agentWork{Checks: []check.Spec{}, Units: []string{}, Fetches: []fetch.Spec{}}
// A check or probe with no `from` goes to every server; a workstation
// gets only those that name it.
ws := srv.workstation(agent)
if dl := srv.cfg.DataLayer; dl != nil {
for _, p := range dl.Probes {
if (len(p.From) == 0 && !ws) || slices.Contains(p.From, agent) {
w.Fetches = append(w.Fetches, p.fetchSpec())
}
}
}
if c := srv.currentChecks(); c != nil {
for _, s := range c.Checks {
if (len(s.From) == 0 && !ws) || slices.Contains(s.From, agent) {
w.Checks = append(w.Checks, s)
}
}
}
for _, u := range srv.currentExpected().Hosts[agent].Units {
w.Units = append(w.Units, units.Name(u))
}
if len(w.Checks) == 0 && len(w.Units) == 0 && len(w.Fetches) == 0 {
return w, ""
}
b, _ := json.Marshal(w)
sum := sha256.Sum256(b)
return w, hex.EncodeToString(sum[:8])
}
func (srv *Server) handleAgentChecks(w http.ResponseWriter, r *http.Request) {
agent, ok := srv.agentFor(r)
if !ok {
http.Error(w, srv.agentRefusal(), http.StatusUnauthorized)
return
}
work, ver := srv.workFor(agent)
writeJSON(w, http.StatusOK, map[string]any{"version": ver, "checks": work.Checks, "units": work.Units, "fetches": work.Fetches})
}
// CheckView is one check as the UI shows it, with every host's latest result.
type CheckView struct {
ID string `json:"id"`
Name string `json:"name"`
Kind string `json:"kind"`
Target string `json:"target"`
Status string `json:"status"` // good, warning, critical, waiting
Word string `json:"word"`
Detail string `json:"detail,omitempty"`
CertDays *int `json:"cert_days,omitempty"`
Results map[string]report.CheckResult `json:"results"` // by host; fresh results only
From []string `json:"from"`
}
// judgeChecks applies the rule that a check is down only when two hosts
// agree (or the only host running it says so). One host failing alone is
// shown, not alerted: it's usually that host's network path.
func (srv *Server) judgeChecks(now time.Time) ([]CheckView, []Condition) {
views := []CheckView{}
var conds []Condition
c := srv.currentChecks()
if c == nil {
return views, nil
}
latest := map[string]map[string]report.CheckResult{} // check → host → result
var hosts []string
for _, h := range srv.store.Snapshot() {
if !srv.workstation(h.name) { // a check with no `from` runs on the servers only
hosts = append(hosts, h.name)
}
if h.seen.IsZero() || now.Sub(h.seen) > srv.cfg.StaleAfter {
continue // a silent host's results are old; its silence alerts on its own
}
for _, res := range h.last.Checks {
if latest[res.ID] == nil {
latest[res.ID] = map[string]report.CheckResult{}
}
latest[res.ID][h.name] = res
}
}
const href = "/checks"
for _, s := range c.Checks {
v := CheckView{ID: s.ID, Name: s.Name, Kind: s.Kind(), Target: s.Target(), Results: map[string]report.CheckResult{}, From: s.From}
if len(v.From) == 0 {
v.From = hosts
}
var failing, passing []string
var soonest time.Time
for host, res := range latest[s.ID] {
if now.Sub(res.At) > 2*s.Every+srv.cfg.StaleAfter {
continue
}
v.Results[host] = res
if res.OK {
passing = append(passing, host)
} else {
failing = append(failing, host)
}
if !res.CertExpires.IsZero() && (soonest.IsZero() || res.CertExpires.Before(soonest)) {
soonest = res.CertExpires
}
}
sort.Strings(failing)
firstErr := ""
if len(failing) > 0 {
firstErr = v.Results[failing[0]].Error
}
n := len(failing) + len(passing)
switch {
case n == 0:
v.Status, v.Word = "waiting", "Waiting for results"
case len(failing) >= 2 || (n == 1 && len(failing) == 1):
v.Status, v.Word = s.Severity, "Down"
v.Detail = fmt.Sprintf("Failing from %s: %s", strings.Join(failing, ", "), firstErr)
conds = append(conds, Condition{Key: "check/" + s.ID, Severity: s.Severity, Title: s.Name + " is down",
Detail: v.Detail + ".", Href: href})
case len(failing) == 1:
v.Status, v.Word = "warning", "Failing from "+failing[0]+" only"
v.Detail = firstErr + ". Not alerting: the other hosts reach it."
default:
v.Status, v.Word = "good", "Up"
}
if !soonest.IsZero() {
days := int(soonest.Sub(now).Hours() / 24)
v.CertDays = &days
sev := ""
switch {
case days < s.TLSCriticalDays:
sev = "critical"
case days < s.TLSWarnDays:
sev = "warning"
}
if sev != "" {
conds = append(conds, Condition{Key: "cert/" + s.ID, Severity: sev,
Title: fmt.Sprintf("%s certificate expires in %s", s.Name, plural(days, "day", "days")),
Detail: fmt.Sprintf("Seen on %s; it expires %s.", s.Target(), soonest.UTC().Format("2006-01-02 15:04 UTC")), Href: href})
if v.Status == "good" {
v.Status, v.Word = sev, "Certificate expiring"
}
}
}
views = append(views, v)
}
return views, conds
}
func (srv *Server) handleChecks(w http.ResponseWriter, r *http.Request) {
views, _ := srv.judgeChecks(time.Now())
src := srv.checksSource()
writeJSON(w, http.StatusOK, map[string]any{"checks": views, "configured": src != "", "source": src, "file": srv.cfg.Checks})
}