Files
cairnobs/internal/server/lead.go
T
jcoffey-dev db156c4f06
release / release (push) Successful in 7m6s
Cairn OBS
The new Cairn OBS: a self-hosted monitor for networks, hosts and
services. Licensed AGPLv3.
2026-11-14 03:27:09 -07:00

154 lines
4.2 KiB
Go

package server
// One evaluator at a time (docs/scale-and-availability.md). When copies
// share a PostgreSQL database, the copy holding the "evaluator" lease is
// the one that judges alerts and sends notices, and runs the jobs that
// write history or send mail; every other copy serves the interface and
// takes reports, reading the evaluator's incidents from the database.
//
// The holder renews every leaseRenew; the lease lapses after leaseTTL, and
// another copy takes it at the next epoch, carrying on from the incidents
// as stored. Incident saves are fenced by the epoch (state.Fence), and a
// round's notices go out only once its saves are in, so a copy that was
// paused and wakes after losing the lease tells nobody. On the way out the
// holder releases the lease and says so, so another takes over at once.
//
// Alone (SQLite, or no database), a copy is always the evaluator.
import (
"context"
"log"
"sync"
"time"
"github.com/cairnobs/cairnobs/internal/state"
)
const (
leaseEvaluator = "evaluator"
leaseTTL = 15 * time.Second
leaseRenew = 5 * time.Second
)
type leadership struct {
mu sync.Mutex
fence *state.Fence // nil: not the evaluator
renewed time.Time
cancel context.CancelFunc // ends the evaluator's jobs
poke chan struct{} // the lease was released: try for it now
}
// leading says whether this copy is the evaluator. A holder that couldn't
// renew in time isn't, whatever it last heard: by then another copy may
// have taken over.
func (srv *Server) leading() bool {
if !srv.shared() {
return true
}
l := &srv.lead
l.mu.Lock()
defer l.mu.Unlock()
return l.fence != nil && time.Since(l.renewed) < leaseTTL
}
// runLead runs the evaluator's jobs for as long as this copy is the
// evaluator, until ctx ends.
func (srv *Server) runLead(ctx context.Context, jobs func(context.Context)) {
if !srv.shared() {
jobs(ctx)
return
}
srv.alerts.onFenced = func() { srv.stepDown("its save was refused") }
t := time.NewTicker(leaseRenew)
defer t.Stop()
for {
srv.tryLead(ctx, jobs)
select {
case <-ctx.Done():
srv.stepDown("")
return
case <-t.C:
case <-srv.lead.poke:
}
}
}
func (srv *Server) tryLead(ctx context.Context, jobs func(context.Context)) {
now := time.Now()
f, ok, err := srv.db.Acquire(leaseEvaluator, srv.copyID, leaseTTL, now)
l := &srv.lead
l.mu.Lock()
cur, lapsed := l.fence, time.Since(l.renewed) >= leaseTTL
switch {
case err != nil:
l.mu.Unlock()
log.Printf("evaluator: can't reach the lease: %v", err)
if cur != nil && lapsed {
srv.stepDown("it couldn't renew the lease")
srv.alerts.lead(nil)
}
case ok && cur != nil && cur.Epoch == f.Epoch:
l.renewed = now
l.mu.Unlock()
case ok:
if l.cancel != nil {
l.cancel()
}
l.mu.Unlock()
// The alerter takes the fence before this copy says it's leading,
// so no round runs unfenced.
srv.alerts.lead(&f)
lctx, cancel := context.WithCancel(ctx)
l.mu.Lock()
l.fence, l.renewed, l.cancel = &f, now, cancel
l.mu.Unlock()
log.Printf("evaluator: this copy (%s) is the evaluator now, epoch %d", srv.copyID, f.Epoch)
go jobs(lctx)
default:
l.mu.Unlock()
if cur != nil {
srv.stepDown("another copy holds the lease")
srv.alerts.lead(nil)
}
}
}
// released hears that the evaluator gave the lease up.
func (srv *Server) released() {
select {
case srv.lead.poke <- struct{}{}:
default:
}
}
// stepDown stops being the evaluator; why "" is on the way out, and gives
// the lease up so another copy needn't wait for it to lapse. The release
// happens under the lock, so a second caller on the way out (runLead and
// ListenAndServe both are) returns only once it's done, not while the
// process is about to exit under it. It doesn't touch the alerter, which
// may be the one calling.
func (srv *Server) stepDown(why string) {
l := &srv.lead
l.mu.Lock()
defer l.mu.Unlock()
f := l.fence
l.fence = nil
if l.cancel != nil {
l.cancel()
l.cancel = nil
}
if f == nil {
return
}
if why == "" {
if err := srv.db.Release(*f); err != nil {
log.Printf("evaluator: can't release the lease: %v", err)
return
}
log.Printf("evaluator: this copy gave the lease up on the way out")
srv.publish(noticeLease, "")
return
}
log.Printf("evaluator: this copy isn't the evaluator any more: %s", why)
}