release / release (push) Successful in 7m6s
The new Cairn OBS: a self-hosted monitor for networks, hosts and services. Licensed AGPLv3.
325 lines
9.1 KiB
Go
325 lines
9.1 KiB
Go
package server
|
|
|
|
// The maps, drawn from discovery (docs/maps.md). Each copy builds the graph
|
|
// from the hosts' latest map samples in its store, at most every
|
|
// mapRebuild. The evaluator records the learned edges it sees, once an hour
|
|
// per edge, in map_edges; every copy reads them back to tell an edge seen
|
|
// once from a steady or a fading one.
|
|
|
|
import (
|
|
"context"
|
|
"log"
|
|
"slices"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/cairnobs/cairnobs/internal/mapgraph"
|
|
"github.com/cairnobs/cairnobs/internal/report"
|
|
"github.com/cairnobs/cairnobs/internal/state"
|
|
)
|
|
|
|
const (
|
|
mapRebuild = 10 * time.Second // how stale a copy's graph may get
|
|
mapEdgesRead = time.Minute // how often a copy rereads map_edges
|
|
mapRecord = time.Minute // how often the evaluator records edges
|
|
edgeFading = 14 * 24 * time.Hour
|
|
edgeDropped = 30 * 24 * time.Hour
|
|
)
|
|
|
|
// Edge stages, as the maps draw them.
|
|
const (
|
|
stageDeclared = "declared"
|
|
stageSeen = "seen"
|
|
stageSteady = "steady"
|
|
stageFading = "fading"
|
|
)
|
|
|
|
type mapState struct {
|
|
mu sync.Mutex
|
|
graph *mapgraph.Graph
|
|
edges map[state.EdgeKey]state.EdgeRow // learned edges as stored
|
|
edgesAt time.Time
|
|
|
|
recorded map[state.EdgeKey]int64 // evaluator: the hour each edge was last recorded
|
|
mem map[state.EdgeKey]*state.EdgeRow // without a database
|
|
|
|
fixes *MapFixes // corrections made on the page (mapfixes.go)
|
|
fixMu sync.Mutex // held while a correction is checked and saved
|
|
|
|
svcFirst map[string]time.Time // when each service was first seen, as stored
|
|
svcAt time.Time
|
|
svcRecorded map[string]int64 // evaluator: the hour each service was last recorded
|
|
svcMem map[string]*state.ServiceRow
|
|
}
|
|
|
|
// mapHosts is every host's latest view of itself, for the graph.
|
|
// mapClusters is what the maps take from each cluster: its controllers,
|
|
// VMs, Services, Ingresses and Routes.
|
|
func (srv *Server) mapClusters() []mapgraph.Cluster {
|
|
ws := &srv.wl
|
|
ws.mu.RLock()
|
|
defer ws.mu.RUnlock()
|
|
out := make([]mapgraph.Cluster, 0, len(ws.src))
|
|
for name, s := range ws.src {
|
|
c := mapgraph.Cluster{Name: name}
|
|
for _, r := range s.recs {
|
|
switch r.Kind {
|
|
case report.KindController, report.KindVM, report.KindService, report.KindIngress:
|
|
c.Records = append(c.Records, r.Workload)
|
|
}
|
|
}
|
|
if len(c.Records) > 0 {
|
|
out = append(out, c)
|
|
}
|
|
}
|
|
slices.SortFunc(out, func(a, b mapgraph.Cluster) int { return strings.Compare(a.Name, b.Name) })
|
|
return out
|
|
}
|
|
|
|
// k8sParts is each Kubernetes part's state, for the service map:
|
|
// PartKey(cluster, "k8s:ns/deployment/web") → severity and word.
|
|
func (srv *Server) k8sParts() map[string][2]string {
|
|
ws := &srv.wl
|
|
ws.mu.RLock()
|
|
defer ws.mu.RUnlock()
|
|
out := map[string][2]string{}
|
|
for name, s := range ws.src {
|
|
for _, r := range s.recs {
|
|
switch r.Kind {
|
|
case report.KindController:
|
|
out[mapgraph.PartKey(name, mapgraph.K8sOwner(r.Namespace, r.Name))] = [2]string{stateSeverity(r.State), r.Word}
|
|
case report.KindVM:
|
|
out[mapgraph.PartKey(name, mapgraph.K8sOwner(r.Namespace, "vm/"+r.Name))] = [2]string{stateSeverity(r.State), r.Word}
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (srv *Server) mapHosts() []mapgraph.Host {
|
|
snap := srv.store.Snapshot()
|
|
out := make([]mapgraph.Host, 0, len(snap))
|
|
for _, h := range snap {
|
|
mh := mapgraph.Host{Name: h.name, Map: h.mp, Workstation: srv.cfg.workstation(h.name)}
|
|
if h.last.Containers != nil {
|
|
mh.Containers = h.last.Containers.List
|
|
}
|
|
out = append(out, mh)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// mapCorrections is topology.yaml read as corrections to what's found,
|
|
// with the page's on top.
|
|
func (srv *Server) mapCorrections() mapgraph.Corrections {
|
|
var c mapgraph.Corrections
|
|
srv.mapFixesNow().apply(&c)
|
|
t := srv.currentTopology()
|
|
if t == nil {
|
|
return c
|
|
}
|
|
c.SiteName = t.Network.Site
|
|
c.Roles = map[string]string{}
|
|
for _, h := range t.Network.Hosts {
|
|
c.SiteHosts = append(c.SiteHosts, h.Name)
|
|
if h.Roles != "" {
|
|
c.Roles[h.Name] = h.Roles
|
|
}
|
|
}
|
|
for _, r := range t.Network.Remote {
|
|
if r.Roles != "" {
|
|
c.Roles[r.Name] = r.Roles
|
|
}
|
|
}
|
|
for _, s := range t.Services {
|
|
c.Services = append(c.Services, mapgraph.Declared{ID: s.ID, Name: s.Name, Tier: s.Tier, Needs: s.Needs,
|
|
Containers: s.Containers, Units: s.Units})
|
|
}
|
|
return c
|
|
}
|
|
|
|
// mapGraph is this copy's graph, rebuilt when it's older than mapRebuild.
|
|
func (srv *Server) mapGraph(now time.Time) *mapgraph.Graph {
|
|
srv.maps.mu.Lock()
|
|
g := srv.maps.graph
|
|
srv.maps.mu.Unlock()
|
|
if g != nil && now.Sub(g.Built) < mapRebuild && !now.Before(g.Built) {
|
|
return g
|
|
}
|
|
g = mapgraph.Build(srv.mapHosts(), srv.mapCorrections(), now, srv.mapClusters()...)
|
|
srv.maps.mu.Lock()
|
|
srv.maps.graph = g
|
|
srv.maps.mu.Unlock()
|
|
return g
|
|
}
|
|
|
|
// learnedEdges is what map_edges holds, reread when older than mapEdgesRead.
|
|
func (srv *Server) learnedEdges(now time.Time) map[state.EdgeKey]state.EdgeRow {
|
|
srv.maps.mu.Lock()
|
|
defer srv.maps.mu.Unlock()
|
|
if srv.db == nil {
|
|
out := make(map[state.EdgeKey]state.EdgeRow, len(srv.maps.mem))
|
|
for k, r := range srv.maps.mem {
|
|
out[k] = *r
|
|
}
|
|
return out
|
|
}
|
|
if srv.maps.edges != nil && now.Sub(srv.maps.edgesAt) < mapEdgesRead {
|
|
return srv.maps.edges
|
|
}
|
|
rows, err := srv.db.Edges()
|
|
if err != nil {
|
|
log.Printf("maps: can't read the learned edges: %v", err)
|
|
if srv.maps.edges == nil {
|
|
srv.maps.edges = map[state.EdgeKey]state.EdgeRow{}
|
|
}
|
|
return srv.maps.edges
|
|
}
|
|
m := make(map[state.EdgeKey]state.EdgeRow, len(rows))
|
|
for _, r := range rows {
|
|
m[r.EdgeKey] = r
|
|
}
|
|
srv.maps.edges, srv.maps.edgesAt = m, now
|
|
return m
|
|
}
|
|
|
|
// recordEdges is the evaluator's: every mapRecord, note each learned edge
|
|
// it sees, at most once an hour each, and once a day drop the ones not
|
|
// seen for edgeDropped.
|
|
func (srv *Server) recordEdges(ctx context.Context) {
|
|
t := time.NewTicker(mapRecord)
|
|
defer t.Stop()
|
|
var purged time.Time
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case now := <-t.C:
|
|
srv.recordEdgesOnce(now)
|
|
if srv.db != nil && now.Sub(purged) > 24*time.Hour {
|
|
if n, err := srv.db.PurgeEdges(now.Add(-edgeDropped)); err != nil {
|
|
log.Printf("maps: can't drop old edges: %v", err)
|
|
} else if n > 0 {
|
|
log.Printf("maps: dropped %d edges not seen for %s", n, edgeDropped)
|
|
}
|
|
if _, err := srv.db.PurgeServices(now.Add(-edgeDropped)); err != nil {
|
|
log.Printf("maps: can't drop old services: %v", err)
|
|
}
|
|
purged = now
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (srv *Server) recordEdgesOnce(now time.Time) {
|
|
g := srv.mapGraph(now)
|
|
srv.recordServices(g, now)
|
|
hour := now.Unix() / 3600
|
|
srv.maps.mu.Lock()
|
|
if srv.maps.recorded == nil {
|
|
srv.maps.recorded = map[state.EdgeKey]int64{}
|
|
}
|
|
var due []state.EdgeKey
|
|
for _, e := range g.Edges {
|
|
if e.Source != "learned" {
|
|
continue
|
|
}
|
|
k := state.EdgeKey{Src: e.From, Dst: e.To, Port: e.Port}
|
|
if srv.maps.recorded[k] != hour {
|
|
due = append(due, k)
|
|
}
|
|
}
|
|
if srv.db == nil {
|
|
if srv.maps.mem == nil {
|
|
srv.maps.mem = map[state.EdgeKey]*state.EdgeRow{}
|
|
}
|
|
for _, k := range due {
|
|
r := srv.maps.mem[k]
|
|
if r == nil {
|
|
r = &state.EdgeRow{EdgeKey: k}
|
|
srv.maps.mem[k] = r
|
|
}
|
|
r.Seen(now)
|
|
srv.maps.recorded[k] = hour
|
|
}
|
|
for k, r := range srv.maps.mem {
|
|
if now.Sub(r.Last) > edgeDropped {
|
|
delete(srv.maps.mem, k)
|
|
}
|
|
}
|
|
srv.maps.mu.Unlock()
|
|
return
|
|
}
|
|
srv.maps.mu.Unlock()
|
|
if len(due) == 0 {
|
|
return
|
|
}
|
|
if err := srv.db.SeenEdges(due, now); err != nil {
|
|
log.Printf("maps: can't record %d edges: %v", len(due), err)
|
|
return
|
|
}
|
|
srv.maps.mu.Lock()
|
|
for _, k := range due {
|
|
srv.maps.recorded[k] = hour
|
|
}
|
|
srv.maps.edgesAt = time.Time{} // reread on the next look
|
|
srv.maps.mu.Unlock()
|
|
}
|
|
|
|
// edgeStage says how far to believe a learned edge.
|
|
func edgeStage(r state.EdgeRow, ok bool, now time.Time) string {
|
|
switch {
|
|
case !ok:
|
|
return stageSeen // seen now, not recorded yet
|
|
case now.Sub(r.Last) > edgeFading:
|
|
return stageFading
|
|
case !r.Steady.IsZero():
|
|
return stageSteady
|
|
}
|
|
return stageSeen
|
|
}
|
|
|
|
// MapEdge is an edge with its stage, as the maps get it.
|
|
type MapEdge struct {
|
|
mapgraph.Edge
|
|
Stage string `json:"stage"`
|
|
First *time.Time `json:"first_seen,omitempty"`
|
|
Last *time.Time `json:"last_seen,omitempty"`
|
|
}
|
|
|
|
// mapEdges is the graph's edges with their stages, plus learned edges not
|
|
// seen now but within edgeDropped, between services that still exist.
|
|
func (srv *Server) mapEdges(g *mapgraph.Graph, now time.Time) []MapEdge {
|
|
rows := srv.learnedEdges(now)
|
|
cor := srv.mapCorrections()
|
|
have := map[string]bool{}
|
|
for _, s := range g.Services {
|
|
have[s.ID] = true
|
|
}
|
|
seen := map[state.EdgeKey]bool{}
|
|
out := make([]MapEdge, 0, len(g.Edges))
|
|
for _, e := range g.Edges {
|
|
me := MapEdge{Edge: e, Stage: stageDeclared}
|
|
k := state.EdgeKey{Src: e.From, Dst: e.To, Port: e.Port}
|
|
seen[k] = true // a confirmed edge is declared now, not drawn twice
|
|
if e.Source == "learned" {
|
|
r, ok := rows[k]
|
|
me.Stage = edgeStage(r, ok, now)
|
|
if ok {
|
|
me.First, me.Last = &r.First, &r.Last
|
|
}
|
|
}
|
|
out = append(out, me)
|
|
}
|
|
for k, r := range rows {
|
|
if seen[k] || !have[k.Src] || !have[k.Dst] || now.Sub(r.Last) > edgeDropped ||
|
|
cor.Rejects(mapgraph.Edge{From: k.Src, To: k.Dst, Port: k.Port}) {
|
|
continue
|
|
}
|
|
out = append(out, MapEdge{Edge: mapgraph.Edge{From: k.Src, To: k.Dst, Port: k.Port, Source: "learned"},
|
|
Stage: edgeStage(r, true, now), First: &r.First, Last: &r.Last})
|
|
}
|
|
return out
|
|
}
|