Files
cairnobs/internal/clusterwatch/watch.go
T
jcoffey-dev db156c4f06
release / release (push) Successful in 7m6s
Cairn OBS
The new Cairn OBS: a self-hosted monitor for networks, hosts and
services. Licensed AGPLv3.
2026-11-14 03:27:09 -07:00

234 lines
6.2 KiB
Go

// Package clusterwatch is the cluster agent (docs/cluster-watcher.md): it
// lists and watches a cluster's nodes, pods, controllers, jobs and volumes
// through the API, keeps the few fields it needs, and sends obs-beacond
// short records: everything when it starts and every 30 minutes, the
// changes every 5 seconds in between.
package clusterwatch
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"math/rand/v2"
"net/http"
"sync"
"time"
"github.com/cairnobs/cairnobs/internal/kube"
)
// API is the part of the Kubernetes API the watcher uses (*kube.Client).
type API interface {
Get(ctx context.Context, path string, v any) error
List(ctx context.Context, path string, each func(raw json.RawMessage) error) (string, error)
Watch(ctx context.Context, path, rv string, timeout time.Duration, fn func(typ string, raw json.RawMessage) error) (string, error)
Version(ctx context.Context) (string, error)
}
// watchFor is how long the agent asks each watch to last; the API ends it
// then, and the agent watches on from where it was.
const watchFor = 5 * time.Minute
// notServedRetry is how often a kind the API doesn't serve is looked for
// again: OpenShift's on a plain Kubernetes cluster, or a CRD not installed
// yet. A var for the tests.
var notServedRetry = 10 * time.Minute
// store is what the watches keep.
type store struct {
mu sync.Mutex
objs view
synced map[string]bool // listed at least once (or can't be: see problems)
problem map[string]string // resource → why it can't be read now
version string
apiDown string // why the API can't be reached, after a minute of trying
failing time.Time // since when /version has failed; zero: it hasn't
usageAt time.Time // when nodemetrics and podmetrics were read; zero: no usage
changes chan struct{}
}
func newStore() *store {
s := &store{objs: view{}, synced: map[string]bool{}, problem: map[string]string{}, changes: make(chan struct{}, 1)}
for _, r := range resources {
s.objs[r.name] = map[string]any{}
}
s.objs["guestfs"] = map[string]any{}
s.objs[tlsSeenRes] = map[string]any{}
return s
}
func (s *store) changed() {
select {
case s.changes <- struct{}{}:
default:
}
}
// snapshot is a shallow copy of every object: the objects themselves are
// never changed once decoded, only replaced.
func (s *store) snapshot() view {
s.mu.Lock()
defer s.mu.Unlock()
v := make(view, len(s.objs))
for k, m := range s.objs {
c := make(map[string]any, len(m))
for id, o := range m {
c[id] = o
}
v[k] = c
}
return v
}
// allSynced says whether every resource has been listed (or can't be).
func (s *store) allSynced() bool {
s.mu.Lock()
defer s.mu.Unlock()
for _, r := range resources {
if !s.synced[r.name] {
return false
}
}
return true
}
func (s *store) setProblem(res string, err error) {
s.mu.Lock()
defer s.mu.Unlock()
was := s.problem[res]
if err == nil {
delete(s.problem, res)
if was != "" {
log.Printf("cluster: reading %s again", res)
}
return
}
msg := err.Error()
switch kube.Code(err) {
case http.StatusForbidden:
msg = "not allowed (the ClusterRole doesn't grant it)"
case http.StatusNotFound:
// Not served by this cluster (an old API version): nothing to read.
s.synced[res] = true
return
}
s.problem[res] = msg
// Something that can't be read counts as listed, so the rest is sent;
// the cluster's record says what's missing.
s.synced[res] = true
if msg != was {
log.Printf("cluster: can't read %s: %v", res, err)
}
}
// watchResource lists one resource, then watches it, listing again when
// the watch can't go on from where it was. It runs until ctx ends.
func watchResource(ctx context.Context, api API, s *store, r resource) {
backoff := time.Second
for ctx.Err() == nil {
fresh := map[string]any{}
rv, err := api.List(ctx, r.path, func(raw json.RawMessage) error {
id, o, err := r.decode(raw)
if err != nil {
return fmt.Errorf("decoding %s: %w", r.name, err)
}
fresh[id] = o
return nil
})
if err != nil {
if ctx.Err() != nil {
return
}
s.setProblem(r.name, err)
if kube.Code(err) == http.StatusNotFound {
sleep(ctx, notServedRetry) // not this kind of cluster, or not yet
continue
}
sleep(ctx, backoff)
backoff = min(backoff*2, time.Minute)
continue
}
s.mu.Lock()
s.objs[r.name] = fresh
s.synced[r.name] = true
s.mu.Unlock()
s.setProblem(r.name, nil)
s.changed()
backoff = time.Second
for ctx.Err() == nil {
// Spread the watches' ends so they don't all start again at once.
d := watchFor + time.Duration(rand.Int64N(int64(time.Minute)))
began := time.Now()
rv, err = api.Watch(ctx, r.path, rv, d, func(typ string, raw json.RawMessage) error {
id, o, err := r.decode(raw)
if err != nil {
return fmt.Errorf("decoding %s: %w", r.name, err)
}
s.mu.Lock()
if typ == kube.Deleted {
delete(s.objs[r.name], id)
} else {
s.objs[r.name][id] = o
}
s.mu.Unlock()
s.changed()
return nil
})
if errors.Is(err, kube.ErrGone) {
break // list again
}
if err != nil && ctx.Err() == nil {
s.setProblem(r.name, err)
sleep(ctx, backoff)
backoff = min(backoff*2, time.Minute)
break // list again: changes may have been missed
}
backoff = time.Second
if time.Since(began) < time.Second {
sleep(ctx, time.Second) // ended at once: don't spin
}
}
}
}
// watchVersion asks for the API's version every minute: it's the
// cluster's version, and when it fails for a minute the API is down.
func watchVersion(ctx context.Context, api API, s *store) {
for ctx.Err() == nil {
v, err := api.Version(ctx)
now := time.Now()
s.mu.Lock()
if err == nil {
if s.apiDown != "" {
log.Printf("cluster: the API answers again")
}
s.version, s.apiDown, s.failing = v, "", time.Time{}
} else if ctx.Err() == nil {
if s.failing.IsZero() {
s.failing = now
}
if now.Sub(s.failing) >= time.Minute && s.apiDown == "" {
s.apiDown = err.Error()
log.Printf("cluster: can't reach the API: %v", err)
}
}
s.mu.Unlock()
wait := time.Minute
if err != nil {
wait = 15 * time.Second
}
sleep(ctx, wait)
}
}
func sleep(ctx context.Context, d time.Duration) {
t := time.NewTimer(d)
defer t.Stop()
select {
case <-ctx.Done():
case <-t.C:
}
}