Files
cairnobs/internal/server/checks.go
T
jcoffey-dev 2abab8264d
ci / check (pull_request) Successful in 3m52s
ci / postgres (pull_request) Successful in 7m37s
Certificates: critical only in the last 48 hours
A certificate four days from its end was critical, so it repeated every
15 minutes at the same level as an outage. Now every certificate
condition carries its end, and one pass holds it at warning until it's
within certificates.critical_within (default 48h) of it, whichever source
judged it critical: hosts' ports and files, site checks, Cloudflare,
Akamai CPS, Vault and cert-manager. An ended one stays critical. The
Certificates page judges by the same window; a certificate only in
files still says nothing until 7 days out, and warns from there.

Closes #34.
2026-11-15 22:36:39 -07:00

290 lines
8.5 KiB
Go

package server
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"net/http"
"os"
"slices"
"sort"
"strings"
"sync"
"time"
"gopkg.in/yaml.v3"
"github.com/cairnobs/cairnobs/internal/check"
"github.com/cairnobs/cairnobs/internal/fetch"
"github.com/cairnobs/cairnobs/internal/report"
"github.com/cairnobs/cairnobs/internal/units"
)
// Checks is checks.yaml: the active checks agents run.
type Checks struct {
Checks []check.Spec `yaml:"checks"`
}
// LoadChecks reads and checks a checks file.
func LoadChecks(p string, agents []AgentConfig) (*Checks, error) {
b, err := os.ReadFile(p)
if err != nil {
return nil, err
}
c, err := parseChecks(b, agentNames(agents))
if err != nil {
return nil, fmt.Errorf("%s: %w", p, err)
}
return c, nil
}
// parseChecks reads a checks document and checks it. Every host in a
// check's `from` must be in known; nil known skips that.
func parseChecks(b []byte, known map[string]bool) (*Checks, error) {
var c Checks
dec := yaml.NewDecoder(bytes.NewReader(b))
dec.KnownFields(true)
if err := dec.Decode(&c); err != nil && !errors.Is(err, io.EOF) {
return nil, err
}
if err := c.check(known); err != nil {
return nil, err
}
return &c, nil
}
// check fills in each check's defaults and refuses a list that can't run.
func (c *Checks) check(known map[string]bool) error {
ids := map[string]bool{}
for i := range c.Checks {
s := &c.Checks[i]
if err := s.Check(); err != nil {
return err
}
if ids[s.ID] {
return fmt.Errorf("two checks called %q", s.ID)
}
ids[s.ID] = true
if known == nil {
continue
}
for _, h := range s.From {
if !known[h] {
return fmt.Errorf("check %s: no host called %q", s.ID, h)
}
}
}
return nil
}
func agentNames(agents []AgentConfig) map[string]bool {
known := map[string]bool{}
for _, a := range agents {
known[a.Name] = true
}
return known
}
// checksFile keeps checks.yaml current: re-read when it changes on disk; a
// file that doesn't parse is logged and the last good list stays. Without
// a file, the checks are edited in the UI and kept in the database (raw is
// what was entered, before defaults; settings.go).
type checksFile struct {
mu sync.Mutex
path string
stat os.FileInfo
cur *Checks
raw []check.Spec
}
func (srv *Server) currentChecks() *Checks {
f := &srv.checks
f.mu.Lock()
defer f.mu.Unlock()
if f.path == "" {
return f.cur
}
st, err := os.Stat(f.path)
if err != nil || (f.stat != nil && st.ModTime().Equal(f.stat.ModTime()) && st.Size() == f.stat.Size()) {
return f.cur
}
first := f.stat == nil
f.stat = st
c, err := LoadChecks(f.path, srv.agentList())
if err != nil {
log.Printf("checks: %v; keeping the last good list", err)
return f.cur
}
f.cur = c
if !first {
srv.audit("obs-beacond", "reloaded the checks", fmt.Sprintf("%s (%d checks)", f.path, len(c.Checks)))
}
return c
}
// agentWork is what an agent is asked to do beyond its own report: the
// checks it runs and the systemd units it watches.
type agentWork struct {
Checks []check.Spec `json:"checks"`
Units []string `json:"units"`
Fetches []fetch.Spec `json:"fetches"`
}
// workFor is an agent's work and its version (empty: nothing to do).
func (srv *Server) workFor(agent string) (agentWork, string) {
w := agentWork{Checks: []check.Spec{}, Units: []string{}, Fetches: []fetch.Spec{}}
// A check or probe with no `from` goes to every server; a workstation
// gets only those that name it.
ws := srv.workstation(agent)
if dl := srv.cfg.DataLayer; dl != nil {
for _, p := range dl.Probes {
if (len(p.From) == 0 && !ws) || slices.Contains(p.From, agent) {
w.Fetches = append(w.Fetches, p.fetchSpec())
}
}
}
if c := srv.currentChecks(); c != nil {
for _, s := range c.Checks {
if (len(s.From) == 0 && !ws) || slices.Contains(s.From, agent) {
w.Checks = append(w.Checks, s)
}
}
}
for _, u := range srv.currentExpected().Hosts[agent].Units {
w.Units = append(w.Units, units.Name(u))
}
if len(w.Checks) == 0 && len(w.Units) == 0 && len(w.Fetches) == 0 {
return w, ""
}
b, _ := json.Marshal(w)
sum := sha256.Sum256(b)
return w, hex.EncodeToString(sum[:8])
}
func (srv *Server) handleAgentChecks(w http.ResponseWriter, r *http.Request) {
agent, ok := srv.agentFor(r)
if !ok {
http.Error(w, srv.agentRefusal(), http.StatusUnauthorized)
return
}
work, ver := srv.workFor(agent)
writeJSON(w, http.StatusOK, map[string]any{"version": ver, "checks": work.Checks, "units": work.Units, "fetches": work.Fetches})
}
// CheckView is one check as the UI shows it, with every host's latest result.
type CheckView struct {
ID string `json:"id"`
Name string `json:"name"`
Kind string `json:"kind"`
Target string `json:"target"`
Status string `json:"status"` // good, warning, critical, waiting
Word string `json:"word"`
Detail string `json:"detail,omitempty"`
CertDays *int `json:"cert_days,omitempty"`
Results map[string]report.CheckResult `json:"results"` // by host; fresh results only
From []string `json:"from"`
}
// judgeChecks applies the rule that a check is down only when two hosts
// agree (or the only host running it says so). One host failing alone is
// shown, not alerted: it's usually that host's network path.
func (srv *Server) judgeChecks(now time.Time) ([]CheckView, []Condition) {
views := []CheckView{}
var conds []Condition
c := srv.currentChecks()
if c == nil {
return views, nil
}
latest := map[string]map[string]report.CheckResult{} // check → host → result
var hosts []string
for _, h := range srv.store.Snapshot() {
if !srv.workstation(h.name) { // a check with no `from` runs on the servers only
hosts = append(hosts, h.name)
}
if h.seen.IsZero() || now.Sub(h.seen) > srv.cfg.StaleAfter {
continue // a silent host's results are old; its silence alerts on its own
}
for _, res := range h.last.Checks {
if latest[res.ID] == nil {
latest[res.ID] = map[string]report.CheckResult{}
}
latest[res.ID][h.name] = res
}
}
const href = "/checks"
for _, s := range c.Checks {
v := CheckView{ID: s.ID, Name: s.Name, Kind: s.Kind(), Target: s.Target(), Results: map[string]report.CheckResult{}, From: s.From}
if len(v.From) == 0 {
v.From = hosts
}
var failing, passing []string
var soonest time.Time
for host, res := range latest[s.ID] {
if now.Sub(res.At) > 2*s.Every+srv.cfg.StaleAfter {
continue
}
v.Results[host] = res
if res.OK {
passing = append(passing, host)
} else {
failing = append(failing, host)
}
if !res.CertExpires.IsZero() && (soonest.IsZero() || res.CertExpires.Before(soonest)) {
soonest = res.CertExpires
}
}
sort.Strings(failing)
firstErr := ""
if len(failing) > 0 {
firstErr = v.Results[failing[0]].Error
}
n := len(failing) + len(passing)
switch {
case n == 0:
v.Status, v.Word = "waiting", "Waiting for results"
case len(failing) >= 2 || (n == 1 && len(failing) == 1):
v.Status, v.Word = s.Severity, "Down"
v.Detail = fmt.Sprintf("Failing from %s: %s", strings.Join(failing, ", "), firstErr)
conds = append(conds, Condition{Key: "check/" + s.ID, Severity: s.Severity, Title: s.Name + " is down",
Detail: v.Detail + ".", Href: href})
case len(failing) == 1:
v.Status, v.Word = "warning", "Failing from "+failing[0]+" only"
v.Detail = firstErr + ". Not alerting: the other hosts reach it."
default:
v.Status, v.Word = "good", "Up"
}
if !soonest.IsZero() {
days := int(soonest.Sub(now).Hours() / 24)
v.CertDays = &days
sev := ""
switch {
case days < s.TLSCriticalDays:
sev = "critical"
case days < s.TLSWarnDays:
sev = "warning"
}
if sev != "" {
conds = append(conds, Condition{Key: "cert/" + s.ID, Severity: sev, Ends: soonest,
Title: fmt.Sprintf("%s certificate expires in %s", s.Name, plural(days, "day", "days")),
Detail: fmt.Sprintf("Seen on %s; it expires %s.", s.Target(), soonest.UTC().Format("2006-01-02 15:04 UTC")), Href: href})
if v.Status == "good" {
v.Status, v.Word = sev, "Certificate expiring"
}
}
}
views = append(views, v)
}
return views, conds
}
func (srv *Server) handleChecks(w http.ResponseWriter, r *http.Request) {
views, _ := srv.judgeChecks(time.Now())
src := srv.checksSource()
writeJSON(w, http.StatusOK, map[string]any{"checks": views, "configured": src != "", "source": src, "file": srv.cfg.Checks})
}