Files
cairnobs/internal/watch/watch.go
T
jcoffey-dev db156c4f06
release / release (push) Successful in 7m6s
Cairn OBS
The new Cairn OBS: a self-hosted monitor for networks, hosts and
services. Licensed AGPLv3.
2026-11-14 03:27:09 -07:00

172 lines
5.4 KiB
Go

// Package watch is "who watches Cairn OBS": an agent on another host notices
// when obs-beacond itself is gone and says so through its own alert channels,
// since obs-beacond can't report its own death.
package watch
import (
"context"
"errors"
"fmt"
"io"
"log"
"net"
"net/http"
"net/url"
"os"
"strings"
"time"
"github.com/cairnobs/cairnobs/internal/maint"
"github.com/cairnobs/cairnobs/internal/notify"
"github.com/cairnobs/cairnobs/internal/version"
)
// Config is the agent's `watch_server` section.
type Config struct {
After time.Duration `yaml:"after"` // reports failing this long before looking closer; default 3m
PublicURL string `yaml:"public_url"` // obs-beacond's health page by its public name, e.g. https://beacon.example.org/healthz
Channels []notify.Config `yaml:"channels"` // same format as obs-beacond's alerts.channels
Quiet []maint.Window `yaml:"quiet"` // obs-beacond's host's own maintenance (its nightly reboot)
}
// Watch decides when obs-beacond is down. Reports failing alone isn't enough:
// that may be this host's link. obs-beacond is down when this host can't
// report AND its public health page doesn't answer either; otherwise
// obs-beacond is up and alerts about this host going quiet by itself.
type Watch struct {
cfg Config
host string
chans []notify.Channel
probe func(ctx context.Context) error
lastOK time.Time
down time.Time // when the outage was announced; zero when all is well
lastDown time.Time // the last report that got through before it
}
// New makes a watch for this host, started now.
func New(cfg Config, host string, now time.Time) (*Watch, error) {
if cfg.After == 0 {
cfg.After = 3 * time.Minute
}
if cfg.PublicURL == "" {
return nil, errors.New("watch_server: public_url is required")
}
if len(cfg.Channels) == 0 {
return nil, errors.New("watch_server: at least one channel is required")
}
for i := range cfg.Quiet {
if err := cfg.Quiet[i].Check(); err != nil {
return nil, fmt.Errorf("watch_server: %w", err)
}
}
w := &Watch{cfg: cfg, host: host, lastOK: now}
for _, c := range cfg.Channels {
ch, err := notify.New(c)
if err != nil {
return nil, fmt.Errorf("watch_server: %w", err)
}
w.chans = append(w.chans, ch)
}
client := &http.Client{Timeout: 10 * time.Second}
w.probe = func(ctx context.Context) error {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, cfg.PublicURL, nil)
if err != nil {
return err
}
req.Header.Set("User-Agent", "obs-beacon-agent/"+version.Version)
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
b, _ := io.ReadAll(io.LimitReader(resp.Body, 256))
if resp.StatusCode != http.StatusOK || !strings.HasPrefix(string(b), "ok") {
return fmt.Errorf("HTTP %d", resp.StatusCode)
}
return nil
}
return w, nil
}
// Reported is called after every report that reached obs-beacond.
func (w *Watch) Reported(ctx context.Context, now time.Time) {
w.lastOK = now
if !w.down.IsZero() {
w.send(ctx, notify.Notice{Kind: "resolved", Severity: "critical", Title: "Cairn OBS is down",
Detail: fmt.Sprintf("%s reaches obs-beacond again; it was out for about %s.", w.host, human(now.Sub(w.lastDown))),
Host: w.host, Since: w.down})
w.down = time.Time{}
}
}
// Failed is called after a report that didn't reach obs-beacond.
func (w *Watch) Failed(ctx context.Context, now time.Time) {
if !w.down.IsZero() || now.Sub(w.lastOK) < w.cfg.After || maint.AnyActive(w.cfg.Quiet, now) {
return // still down after the window closes: it alerts then
}
pctx, cancel := context.WithTimeout(ctx, 15*time.Second)
err := w.probe(pctx)
cancel()
if err == nil {
return // obs-beacond is up; it will say this host went quiet
}
if w.send(ctx, notify.Notice{Kind: "firing", Severity: "critical", Title: "Cairn OBS is down",
Detail: fmt.Sprintf("%s hasn't reached obs-beacond for %s, and %s doesn't answer either (%s). Nothing else will alert until it's back.",
w.host, human(now.Sub(w.lastOK)), w.cfg.PublicURL, plain(err)),
Host: w.host, Since: w.lastOK}) {
w.down, w.lastDown = now, w.lastOK
}
}
// send tells every channel that takes the severity; true if at least one
// took it (otherwise the next report tries again).
func (w *Watch) send(ctx context.Context, n notify.Notice) bool {
ok := false
for _, ch := range w.chans {
if notify.Rank(n.Severity) < notify.Rank(ch.Min()) {
continue
}
sctx, cancel := context.WithTimeout(ctx, 15*time.Second)
err := ch.Send(sctx, n)
cancel()
if err != nil {
log.Printf("watch: %s to %s failed: %v", n.Kind, ch.ID(), err)
continue
}
log.Printf("watch: sent %s %q to %s", n.Kind, n.Title, ch.ID())
ok = true
}
return ok
}
// human is a duration as a person says it: 45 s, 3 min, 2 h 5 min.
func human(d time.Duration) string {
switch {
case d < time.Minute:
return fmt.Sprintf("%d s", int(d.Seconds()))
case d < time.Hour:
return fmt.Sprintf("%d min", int(d.Minutes()))
}
return fmt.Sprintf("%d h %d min", int(d.Hours()), int(d.Minutes())%60)
}
// plain keeps the part of a network error a person needs.
func plain(err error) string {
var ue *url.Error
if errors.As(err, &ue) {
err = ue.Err
}
var oe *net.OpError
if errors.As(err, &oe) && oe.Err != nil {
err = oe.Err
var se *os.SyscallError
if errors.As(err, &se) {
err = se.Err
}
}
if errors.Is(err, context.DeadlineExceeded) {
return "timed out"
}
return err.Error()
}