release / release (push) Successful in 7m6s
The new Cairn OBS: a self-hosted monitor for networks, hosts and services. Licensed AGPLv3.
172 lines
5.4 KiB
Go
172 lines
5.4 KiB
Go
// Package watch is "who watches Cairn OBS": an agent on another host notices
|
|
// when obs-beacond itself is gone and says so through its own alert channels,
|
|
// since obs-beacond can't report its own death.
|
|
package watch
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/cairnobs/cairnobs/internal/maint"
|
|
"github.com/cairnobs/cairnobs/internal/notify"
|
|
"github.com/cairnobs/cairnobs/internal/version"
|
|
)
|
|
|
|
// Config is the agent's `watch_server` section.
|
|
type Config struct {
|
|
After time.Duration `yaml:"after"` // reports failing this long before looking closer; default 3m
|
|
PublicURL string `yaml:"public_url"` // obs-beacond's health page by its public name, e.g. https://beacon.example.org/healthz
|
|
Channels []notify.Config `yaml:"channels"` // same format as obs-beacond's alerts.channels
|
|
Quiet []maint.Window `yaml:"quiet"` // obs-beacond's host's own maintenance (its nightly reboot)
|
|
}
|
|
|
|
// Watch decides when obs-beacond is down. Reports failing alone isn't enough:
|
|
// that may be this host's link. obs-beacond is down when this host can't
|
|
// report AND its public health page doesn't answer either; otherwise
|
|
// obs-beacond is up and alerts about this host going quiet by itself.
|
|
type Watch struct {
|
|
cfg Config
|
|
host string
|
|
chans []notify.Channel
|
|
probe func(ctx context.Context) error
|
|
lastOK time.Time
|
|
down time.Time // when the outage was announced; zero when all is well
|
|
lastDown time.Time // the last report that got through before it
|
|
}
|
|
|
|
// New makes a watch for this host, started now.
|
|
func New(cfg Config, host string, now time.Time) (*Watch, error) {
|
|
if cfg.After == 0 {
|
|
cfg.After = 3 * time.Minute
|
|
}
|
|
if cfg.PublicURL == "" {
|
|
return nil, errors.New("watch_server: public_url is required")
|
|
}
|
|
if len(cfg.Channels) == 0 {
|
|
return nil, errors.New("watch_server: at least one channel is required")
|
|
}
|
|
for i := range cfg.Quiet {
|
|
if err := cfg.Quiet[i].Check(); err != nil {
|
|
return nil, fmt.Errorf("watch_server: %w", err)
|
|
}
|
|
}
|
|
w := &Watch{cfg: cfg, host: host, lastOK: now}
|
|
for _, c := range cfg.Channels {
|
|
ch, err := notify.New(c)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("watch_server: %w", err)
|
|
}
|
|
w.chans = append(w.chans, ch)
|
|
}
|
|
client := &http.Client{Timeout: 10 * time.Second}
|
|
w.probe = func(ctx context.Context) error {
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, cfg.PublicURL, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
req.Header.Set("User-Agent", "obs-beacon-agent/"+version.Version)
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
b, _ := io.ReadAll(io.LimitReader(resp.Body, 256))
|
|
if resp.StatusCode != http.StatusOK || !strings.HasPrefix(string(b), "ok") {
|
|
return fmt.Errorf("HTTP %d", resp.StatusCode)
|
|
}
|
|
return nil
|
|
}
|
|
return w, nil
|
|
}
|
|
|
|
// Reported is called after every report that reached obs-beacond.
|
|
func (w *Watch) Reported(ctx context.Context, now time.Time) {
|
|
w.lastOK = now
|
|
if !w.down.IsZero() {
|
|
w.send(ctx, notify.Notice{Kind: "resolved", Severity: "critical", Title: "Cairn OBS is down",
|
|
Detail: fmt.Sprintf("%s reaches obs-beacond again; it was out for about %s.", w.host, human(now.Sub(w.lastDown))),
|
|
Host: w.host, Since: w.down})
|
|
w.down = time.Time{}
|
|
}
|
|
}
|
|
|
|
// Failed is called after a report that didn't reach obs-beacond.
|
|
func (w *Watch) Failed(ctx context.Context, now time.Time) {
|
|
if !w.down.IsZero() || now.Sub(w.lastOK) < w.cfg.After || maint.AnyActive(w.cfg.Quiet, now) {
|
|
return // still down after the window closes: it alerts then
|
|
}
|
|
pctx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
|
err := w.probe(pctx)
|
|
cancel()
|
|
if err == nil {
|
|
return // obs-beacond is up; it will say this host went quiet
|
|
}
|
|
if w.send(ctx, notify.Notice{Kind: "firing", Severity: "critical", Title: "Cairn OBS is down",
|
|
Detail: fmt.Sprintf("%s hasn't reached obs-beacond for %s, and %s doesn't answer either (%s). Nothing else will alert until it's back.",
|
|
w.host, human(now.Sub(w.lastOK)), w.cfg.PublicURL, plain(err)),
|
|
Host: w.host, Since: w.lastOK}) {
|
|
w.down, w.lastDown = now, w.lastOK
|
|
}
|
|
}
|
|
|
|
// send tells every channel that takes the severity; true if at least one
|
|
// took it (otherwise the next report tries again).
|
|
func (w *Watch) send(ctx context.Context, n notify.Notice) bool {
|
|
ok := false
|
|
for _, ch := range w.chans {
|
|
if notify.Rank(n.Severity) < notify.Rank(ch.Min()) {
|
|
continue
|
|
}
|
|
sctx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
|
err := ch.Send(sctx, n)
|
|
cancel()
|
|
if err != nil {
|
|
log.Printf("watch: %s to %s failed: %v", n.Kind, ch.ID(), err)
|
|
continue
|
|
}
|
|
log.Printf("watch: sent %s %q to %s", n.Kind, n.Title, ch.ID())
|
|
ok = true
|
|
}
|
|
return ok
|
|
}
|
|
|
|
// human is a duration as a person says it: 45 s, 3 min, 2 h 5 min.
|
|
func human(d time.Duration) string {
|
|
switch {
|
|
case d < time.Minute:
|
|
return fmt.Sprintf("%d s", int(d.Seconds()))
|
|
case d < time.Hour:
|
|
return fmt.Sprintf("%d min", int(d.Minutes()))
|
|
}
|
|
return fmt.Sprintf("%d h %d min", int(d.Hours()), int(d.Minutes())%60)
|
|
}
|
|
|
|
// plain keeps the part of a network error a person needs.
|
|
func plain(err error) string {
|
|
var ue *url.Error
|
|
if errors.As(err, &ue) {
|
|
err = ue.Err
|
|
}
|
|
var oe *net.OpError
|
|
if errors.As(err, &oe) && oe.Err != nil {
|
|
err = oe.Err
|
|
var se *os.SyscallError
|
|
if errors.As(err, &se) {
|
|
err = se.Err
|
|
}
|
|
}
|
|
if errors.Is(err, context.DeadlineExceeded) {
|
|
return "timed out"
|
|
}
|
|
return err.Error()
|
|
}
|