release / release (push) Successful in 7m6s
The new Cairn OBS: a self-hosted monitor for networks, hosts and services. Licensed AGPLv3.
234 lines
6.2 KiB
Go
234 lines
6.2 KiB
Go
// Package clusterwatch is the cluster agent (docs/cluster-watcher.md): it
|
|
// lists and watches a cluster's nodes, pods, controllers, jobs and volumes
|
|
// through the API, keeps the few fields it needs, and sends obs-beacond
|
|
// short records: everything when it starts and every 30 minutes, the
|
|
// changes every 5 seconds in between.
|
|
package clusterwatch
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"math/rand/v2"
|
|
"net/http"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/cairnobs/cairnobs/internal/kube"
|
|
)
|
|
|
|
// API is the part of the Kubernetes API the watcher uses (*kube.Client).
|
|
type API interface {
|
|
Get(ctx context.Context, path string, v any) error
|
|
List(ctx context.Context, path string, each func(raw json.RawMessage) error) (string, error)
|
|
Watch(ctx context.Context, path, rv string, timeout time.Duration, fn func(typ string, raw json.RawMessage) error) (string, error)
|
|
Version(ctx context.Context) (string, error)
|
|
}
|
|
|
|
// watchFor is how long the agent asks each watch to last; the API ends it
|
|
// then, and the agent watches on from where it was.
|
|
const watchFor = 5 * time.Minute
|
|
|
|
// notServedRetry is how often a kind the API doesn't serve is looked for
|
|
// again: OpenShift's on a plain Kubernetes cluster, or a CRD not installed
|
|
// yet. A var for the tests.
|
|
var notServedRetry = 10 * time.Minute
|
|
|
|
// store is what the watches keep.
|
|
type store struct {
|
|
mu sync.Mutex
|
|
objs view
|
|
synced map[string]bool // listed at least once (or can't be: see problems)
|
|
problem map[string]string // resource → why it can't be read now
|
|
version string
|
|
apiDown string // why the API can't be reached, after a minute of trying
|
|
failing time.Time // since when /version has failed; zero: it hasn't
|
|
usageAt time.Time // when nodemetrics and podmetrics were read; zero: no usage
|
|
changes chan struct{}
|
|
}
|
|
|
|
func newStore() *store {
|
|
s := &store{objs: view{}, synced: map[string]bool{}, problem: map[string]string{}, changes: make(chan struct{}, 1)}
|
|
for _, r := range resources {
|
|
s.objs[r.name] = map[string]any{}
|
|
}
|
|
s.objs["guestfs"] = map[string]any{}
|
|
s.objs[tlsSeenRes] = map[string]any{}
|
|
return s
|
|
}
|
|
|
|
func (s *store) changed() {
|
|
select {
|
|
case s.changes <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
|
|
// snapshot is a shallow copy of every object: the objects themselves are
|
|
// never changed once decoded, only replaced.
|
|
func (s *store) snapshot() view {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
v := make(view, len(s.objs))
|
|
for k, m := range s.objs {
|
|
c := make(map[string]any, len(m))
|
|
for id, o := range m {
|
|
c[id] = o
|
|
}
|
|
v[k] = c
|
|
}
|
|
return v
|
|
}
|
|
|
|
// allSynced says whether every resource has been listed (or can't be).
|
|
func (s *store) allSynced() bool {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
for _, r := range resources {
|
|
if !s.synced[r.name] {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (s *store) setProblem(res string, err error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
was := s.problem[res]
|
|
if err == nil {
|
|
delete(s.problem, res)
|
|
if was != "" {
|
|
log.Printf("cluster: reading %s again", res)
|
|
}
|
|
return
|
|
}
|
|
msg := err.Error()
|
|
switch kube.Code(err) {
|
|
case http.StatusForbidden:
|
|
msg = "not allowed (the ClusterRole doesn't grant it)"
|
|
case http.StatusNotFound:
|
|
// Not served by this cluster (an old API version): nothing to read.
|
|
s.synced[res] = true
|
|
return
|
|
}
|
|
s.problem[res] = msg
|
|
// Something that can't be read counts as listed, so the rest is sent;
|
|
// the cluster's record says what's missing.
|
|
s.synced[res] = true
|
|
if msg != was {
|
|
log.Printf("cluster: can't read %s: %v", res, err)
|
|
}
|
|
}
|
|
|
|
// watchResource lists one resource, then watches it, listing again when
|
|
// the watch can't go on from where it was. It runs until ctx ends.
|
|
func watchResource(ctx context.Context, api API, s *store, r resource) {
|
|
backoff := time.Second
|
|
for ctx.Err() == nil {
|
|
fresh := map[string]any{}
|
|
rv, err := api.List(ctx, r.path, func(raw json.RawMessage) error {
|
|
id, o, err := r.decode(raw)
|
|
if err != nil {
|
|
return fmt.Errorf("decoding %s: %w", r.name, err)
|
|
}
|
|
fresh[id] = o
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
s.setProblem(r.name, err)
|
|
if kube.Code(err) == http.StatusNotFound {
|
|
sleep(ctx, notServedRetry) // not this kind of cluster, or not yet
|
|
continue
|
|
}
|
|
sleep(ctx, backoff)
|
|
backoff = min(backoff*2, time.Minute)
|
|
continue
|
|
}
|
|
s.mu.Lock()
|
|
s.objs[r.name] = fresh
|
|
s.synced[r.name] = true
|
|
s.mu.Unlock()
|
|
s.setProblem(r.name, nil)
|
|
s.changed()
|
|
backoff = time.Second
|
|
for ctx.Err() == nil {
|
|
// Spread the watches' ends so they don't all start again at once.
|
|
d := watchFor + time.Duration(rand.Int64N(int64(time.Minute)))
|
|
began := time.Now()
|
|
rv, err = api.Watch(ctx, r.path, rv, d, func(typ string, raw json.RawMessage) error {
|
|
id, o, err := r.decode(raw)
|
|
if err != nil {
|
|
return fmt.Errorf("decoding %s: %w", r.name, err)
|
|
}
|
|
s.mu.Lock()
|
|
if typ == kube.Deleted {
|
|
delete(s.objs[r.name], id)
|
|
} else {
|
|
s.objs[r.name][id] = o
|
|
}
|
|
s.mu.Unlock()
|
|
s.changed()
|
|
return nil
|
|
})
|
|
if errors.Is(err, kube.ErrGone) {
|
|
break // list again
|
|
}
|
|
if err != nil && ctx.Err() == nil {
|
|
s.setProblem(r.name, err)
|
|
sleep(ctx, backoff)
|
|
backoff = min(backoff*2, time.Minute)
|
|
break // list again: changes may have been missed
|
|
}
|
|
backoff = time.Second
|
|
if time.Since(began) < time.Second {
|
|
sleep(ctx, time.Second) // ended at once: don't spin
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// watchVersion asks for the API's version every minute: it's the
|
|
// cluster's version, and when it fails for a minute the API is down.
|
|
func watchVersion(ctx context.Context, api API, s *store) {
|
|
for ctx.Err() == nil {
|
|
v, err := api.Version(ctx)
|
|
now := time.Now()
|
|
s.mu.Lock()
|
|
if err == nil {
|
|
if s.apiDown != "" {
|
|
log.Printf("cluster: the API answers again")
|
|
}
|
|
s.version, s.apiDown, s.failing = v, "", time.Time{}
|
|
} else if ctx.Err() == nil {
|
|
if s.failing.IsZero() {
|
|
s.failing = now
|
|
}
|
|
if now.Sub(s.failing) >= time.Minute && s.apiDown == "" {
|
|
s.apiDown = err.Error()
|
|
log.Printf("cluster: can't reach the API: %v", err)
|
|
}
|
|
}
|
|
s.mu.Unlock()
|
|
wait := time.Minute
|
|
if err != nil {
|
|
wait = 15 * time.Second
|
|
}
|
|
sleep(ctx, wait)
|
|
}
|
|
}
|
|
|
|
func sleep(ctx context.Context, d time.Duration) {
|
|
t := time.NewTimer(d)
|
|
defer t.Stop()
|
|
select {
|
|
case <-ctx.Done():
|
|
case <-t.C:
|
|
}
|
|
}
|