A certificate four days from its end was critical, so it repeated every 15 minutes at the same level as an outage. Now every certificate condition carries its end, and one pass holds it at warning until it's within certificates.critical_within (default 48h) of it, whichever source judged it critical: hosts' ports and files, site checks, Cloudflare, Akamai CPS, Vault and cert-manager. An ended one stays critical. The Certificates page judges by the same window; a certificate only in files still says nothing until 7 days out, and warns from there. Closes #34.
290 lines
8.5 KiB
Go
290 lines
8.5 KiB
Go
package server
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"net/http"
|
|
"os"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"gopkg.in/yaml.v3"
|
|
|
|
"github.com/cairnobs/cairnobs/internal/check"
|
|
"github.com/cairnobs/cairnobs/internal/fetch"
|
|
"github.com/cairnobs/cairnobs/internal/report"
|
|
"github.com/cairnobs/cairnobs/internal/units"
|
|
)
|
|
|
|
// Checks is checks.yaml: the active checks agents run.
|
|
type Checks struct {
|
|
Checks []check.Spec `yaml:"checks"`
|
|
}
|
|
|
|
// LoadChecks reads and checks a checks file.
|
|
func LoadChecks(p string, agents []AgentConfig) (*Checks, error) {
|
|
b, err := os.ReadFile(p)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
c, err := parseChecks(b, agentNames(agents))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%s: %w", p, err)
|
|
}
|
|
return c, nil
|
|
}
|
|
|
|
// parseChecks reads a checks document and checks it. Every host in a
|
|
// check's `from` must be in known; nil known skips that.
|
|
func parseChecks(b []byte, known map[string]bool) (*Checks, error) {
|
|
var c Checks
|
|
dec := yaml.NewDecoder(bytes.NewReader(b))
|
|
dec.KnownFields(true)
|
|
if err := dec.Decode(&c); err != nil && !errors.Is(err, io.EOF) {
|
|
return nil, err
|
|
}
|
|
if err := c.check(known); err != nil {
|
|
return nil, err
|
|
}
|
|
return &c, nil
|
|
}
|
|
|
|
// check fills in each check's defaults and refuses a list that can't run.
|
|
func (c *Checks) check(known map[string]bool) error {
|
|
ids := map[string]bool{}
|
|
for i := range c.Checks {
|
|
s := &c.Checks[i]
|
|
if err := s.Check(); err != nil {
|
|
return err
|
|
}
|
|
if ids[s.ID] {
|
|
return fmt.Errorf("two checks called %q", s.ID)
|
|
}
|
|
ids[s.ID] = true
|
|
if known == nil {
|
|
continue
|
|
}
|
|
for _, h := range s.From {
|
|
if !known[h] {
|
|
return fmt.Errorf("check %s: no host called %q", s.ID, h)
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func agentNames(agents []AgentConfig) map[string]bool {
|
|
known := map[string]bool{}
|
|
for _, a := range agents {
|
|
known[a.Name] = true
|
|
}
|
|
return known
|
|
}
|
|
|
|
// checksFile keeps checks.yaml current: re-read when it changes on disk; a
|
|
// file that doesn't parse is logged and the last good list stays. Without
|
|
// a file, the checks are edited in the UI and kept in the database (raw is
|
|
// what was entered, before defaults; settings.go).
|
|
type checksFile struct {
|
|
mu sync.Mutex
|
|
path string
|
|
stat os.FileInfo
|
|
cur *Checks
|
|
raw []check.Spec
|
|
}
|
|
|
|
func (srv *Server) currentChecks() *Checks {
|
|
f := &srv.checks
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
if f.path == "" {
|
|
return f.cur
|
|
}
|
|
st, err := os.Stat(f.path)
|
|
if err != nil || (f.stat != nil && st.ModTime().Equal(f.stat.ModTime()) && st.Size() == f.stat.Size()) {
|
|
return f.cur
|
|
}
|
|
first := f.stat == nil
|
|
f.stat = st
|
|
c, err := LoadChecks(f.path, srv.agentList())
|
|
if err != nil {
|
|
log.Printf("checks: %v; keeping the last good list", err)
|
|
return f.cur
|
|
}
|
|
f.cur = c
|
|
if !first {
|
|
srv.audit("obs-beacond", "reloaded the checks", fmt.Sprintf("%s (%d checks)", f.path, len(c.Checks)))
|
|
}
|
|
return c
|
|
}
|
|
|
|
// agentWork is what an agent is asked to do beyond its own report: the
|
|
// checks it runs and the systemd units it watches.
|
|
type agentWork struct {
|
|
Checks []check.Spec `json:"checks"`
|
|
Units []string `json:"units"`
|
|
Fetches []fetch.Spec `json:"fetches"`
|
|
}
|
|
|
|
// workFor is an agent's work and its version (empty: nothing to do).
|
|
func (srv *Server) workFor(agent string) (agentWork, string) {
|
|
w := agentWork{Checks: []check.Spec{}, Units: []string{}, Fetches: []fetch.Spec{}}
|
|
// A check or probe with no `from` goes to every server; a workstation
|
|
// gets only those that name it.
|
|
ws := srv.workstation(agent)
|
|
if dl := srv.cfg.DataLayer; dl != nil {
|
|
for _, p := range dl.Probes {
|
|
if (len(p.From) == 0 && !ws) || slices.Contains(p.From, agent) {
|
|
w.Fetches = append(w.Fetches, p.fetchSpec())
|
|
}
|
|
}
|
|
}
|
|
if c := srv.currentChecks(); c != nil {
|
|
for _, s := range c.Checks {
|
|
if (len(s.From) == 0 && !ws) || slices.Contains(s.From, agent) {
|
|
w.Checks = append(w.Checks, s)
|
|
}
|
|
}
|
|
}
|
|
for _, u := range srv.currentExpected().Hosts[agent].Units {
|
|
w.Units = append(w.Units, units.Name(u))
|
|
}
|
|
if len(w.Checks) == 0 && len(w.Units) == 0 && len(w.Fetches) == 0 {
|
|
return w, ""
|
|
}
|
|
b, _ := json.Marshal(w)
|
|
sum := sha256.Sum256(b)
|
|
return w, hex.EncodeToString(sum[:8])
|
|
}
|
|
|
|
func (srv *Server) handleAgentChecks(w http.ResponseWriter, r *http.Request) {
|
|
agent, ok := srv.agentFor(r)
|
|
if !ok {
|
|
http.Error(w, srv.agentRefusal(), http.StatusUnauthorized)
|
|
return
|
|
}
|
|
work, ver := srv.workFor(agent)
|
|
writeJSON(w, http.StatusOK, map[string]any{"version": ver, "checks": work.Checks, "units": work.Units, "fetches": work.Fetches})
|
|
}
|
|
|
|
// CheckView is one check as the UI shows it, with every host's latest result.
|
|
type CheckView struct {
|
|
ID string `json:"id"`
|
|
Name string `json:"name"`
|
|
Kind string `json:"kind"`
|
|
Target string `json:"target"`
|
|
Status string `json:"status"` // good, warning, critical, waiting
|
|
Word string `json:"word"`
|
|
Detail string `json:"detail,omitempty"`
|
|
CertDays *int `json:"cert_days,omitempty"`
|
|
Results map[string]report.CheckResult `json:"results"` // by host; fresh results only
|
|
From []string `json:"from"`
|
|
}
|
|
|
|
// judgeChecks applies the rule that a check is down only when two hosts
|
|
// agree (or the only host running it says so). One host failing alone is
|
|
// shown, not alerted: it's usually that host's network path.
|
|
func (srv *Server) judgeChecks(now time.Time) ([]CheckView, []Condition) {
|
|
views := []CheckView{}
|
|
var conds []Condition
|
|
c := srv.currentChecks()
|
|
if c == nil {
|
|
return views, nil
|
|
}
|
|
latest := map[string]map[string]report.CheckResult{} // check → host → result
|
|
var hosts []string
|
|
for _, h := range srv.store.Snapshot() {
|
|
if !srv.workstation(h.name) { // a check with no `from` runs on the servers only
|
|
hosts = append(hosts, h.name)
|
|
}
|
|
if h.seen.IsZero() || now.Sub(h.seen) > srv.cfg.StaleAfter {
|
|
continue // a silent host's results are old; its silence alerts on its own
|
|
}
|
|
for _, res := range h.last.Checks {
|
|
if latest[res.ID] == nil {
|
|
latest[res.ID] = map[string]report.CheckResult{}
|
|
}
|
|
latest[res.ID][h.name] = res
|
|
}
|
|
}
|
|
const href = "/checks"
|
|
for _, s := range c.Checks {
|
|
v := CheckView{ID: s.ID, Name: s.Name, Kind: s.Kind(), Target: s.Target(), Results: map[string]report.CheckResult{}, From: s.From}
|
|
if len(v.From) == 0 {
|
|
v.From = hosts
|
|
}
|
|
var failing, passing []string
|
|
var soonest time.Time
|
|
for host, res := range latest[s.ID] {
|
|
if now.Sub(res.At) > 2*s.Every+srv.cfg.StaleAfter {
|
|
continue
|
|
}
|
|
v.Results[host] = res
|
|
if res.OK {
|
|
passing = append(passing, host)
|
|
} else {
|
|
failing = append(failing, host)
|
|
}
|
|
if !res.CertExpires.IsZero() && (soonest.IsZero() || res.CertExpires.Before(soonest)) {
|
|
soonest = res.CertExpires
|
|
}
|
|
}
|
|
sort.Strings(failing)
|
|
firstErr := ""
|
|
if len(failing) > 0 {
|
|
firstErr = v.Results[failing[0]].Error
|
|
}
|
|
n := len(failing) + len(passing)
|
|
switch {
|
|
case n == 0:
|
|
v.Status, v.Word = "waiting", "Waiting for results"
|
|
case len(failing) >= 2 || (n == 1 && len(failing) == 1):
|
|
v.Status, v.Word = s.Severity, "Down"
|
|
v.Detail = fmt.Sprintf("Failing from %s: %s", strings.Join(failing, ", "), firstErr)
|
|
conds = append(conds, Condition{Key: "check/" + s.ID, Severity: s.Severity, Title: s.Name + " is down",
|
|
Detail: v.Detail + ".", Href: href})
|
|
case len(failing) == 1:
|
|
v.Status, v.Word = "warning", "Failing from "+failing[0]+" only"
|
|
v.Detail = firstErr + ". Not alerting: the other hosts reach it."
|
|
default:
|
|
v.Status, v.Word = "good", "Up"
|
|
}
|
|
if !soonest.IsZero() {
|
|
days := int(soonest.Sub(now).Hours() / 24)
|
|
v.CertDays = &days
|
|
sev := ""
|
|
switch {
|
|
case days < s.TLSCriticalDays:
|
|
sev = "critical"
|
|
case days < s.TLSWarnDays:
|
|
sev = "warning"
|
|
}
|
|
if sev != "" {
|
|
conds = append(conds, Condition{Key: "cert/" + s.ID, Severity: sev, Ends: soonest,
|
|
Title: fmt.Sprintf("%s certificate expires in %s", s.Name, plural(days, "day", "days")),
|
|
Detail: fmt.Sprintf("Seen on %s; it expires %s.", s.Target(), soonest.UTC().Format("2006-01-02 15:04 UTC")), Href: href})
|
|
if v.Status == "good" {
|
|
v.Status, v.Word = sev, "Certificate expiring"
|
|
}
|
|
}
|
|
}
|
|
views = append(views, v)
|
|
}
|
|
return views, conds
|
|
}
|
|
|
|
func (srv *Server) handleChecks(w http.ResponseWriter, r *http.Request) {
|
|
views, _ := srv.judgeChecks(time.Now())
|
|
src := srv.checksSource()
|
|
writeJSON(w, http.StatusOK, map[string]any{"checks": views, "configured": src != "", "source": src, "file": srv.cfg.Checks})
|
|
}
|