Files
cairnobs/internal/server/adoptcluster.go
T
jcoffey-dev f71f968c7f
ci / postgres (pull_request) Successful in 3m35s
ci / check (pull_request) Successful in 7m27s
IBM i, steps 2 and 3: subsystems, disks, interfaces, QSYSOPR and job queues
The reader adds SUBSYSTEM_INFO, SYSDISKSTAT, NETSTAT_INTERFACE_INFO,
MESSAGE_QUEUE_INFO for QSYSOPR and JOB_QUEUE_INFO, still views only.
Subsystems are parts, judged against subsystems: in expected.yaml by
obs-beacond; disk units in trouble degrade their ASP; an interface set
to start with TCP/IP that isn't active is degraded or down; QSYSOPR's
unanswered inquiries (a job in MSGW) warn and its messages of severity
90 and up are critical; a held or orphaned job queue with jobs ready
warns. Adopting a cluster workload keeps the subsystems list.
2026-11-14 23:51:03 -07:00

319 lines
9.9 KiB
Go

package server
// Adopting in a cluster (docs/cluster-watcher.md, What pages, and who
// decides). Out of the box only the cluster and its nodes page; a
// workload's problems are warnings on the pages. An admin adopts whole
// namespaces, or single controllers and VMs, on Configure → Clusters → a
// cluster, and those page. Teams can do the same from the cluster with the
// obs.cairnobs.org/alert annotation.
//
// Which wins: a setting in Cairn OBS for the workload itself, then the
// cluster's annotation (the workload's, its controller's or its
// namespace's, as the agent found it), then a setting in Cairn OBS for its
// namespace. So an admin always has the last word on one workload, and a
// team's annotation is respected unless an admin set that very workload.
import (
"cmp"
"encoding/json"
"fmt"
"maps"
"net/http"
"slices"
"strings"
"github.com/cairnobs/cairnobs/internal/report"
)
// adoption is one cluster's adopted namespaces and workloads.
type adoption struct {
ns map[string]string
wl map[string]ExpectedWorkload
list []ExpectedWorkload // in the file's order, for what's missing
platform bool // OpenShift: its own namespaces are off unless set
}
func adoptionFor(e *Expected, cluster string) adoption {
if e == nil {
return adoption{}
}
ec, ok := e.Clusters[cluster]
if !ok {
return adoption{}
}
a := adoption{ns: ec.Namespaces, wl: make(map[string]ExpectedWorkload, len(ec.Workloads)), list: ec.Workloads}
for _, w := range ec.Workloads {
a.wl[w.Name] = w
}
return a
}
// recKey is a workload's name as adopting knows it, namespace/kind/name:
// "shop/deployment/web", "vms/vm/db1". A pod is its controller's.
func recKey(r report.Workload) string {
switch {
case r.Kind == report.KindController:
return r.Namespace + "/" + r.Name
case r.Kind == report.KindPod && r.Owner != "":
return r.Namespace + "/" + r.Owner
}
return r.Namespace + "/" + r.Kind + "/" + r.Name
}
// alert is the alert setting in force for a record; annotated is what the
// agent found on the cluster.
func (a adoption) alert(r report.Workload, annotated string) string {
if r.Namespace == "" { // the cluster and its nodes: the annotation only
return annotated
}
return a.alertKey(r.Namespace, recKey(r), annotated)
}
func (a adoption) alertKey(ns, key, annotated string) string {
if w, ok := a.wl[key]; ok {
if w.Alert == "" {
return report.AlertPage
}
return w.Alert
}
if annotated != "" {
return annotated
}
if v, ok := a.ns[ns]; ok {
return v
}
if a.platform && platformNamespace(ns) {
return report.AlertOff // judged through the ClusterOperators
}
return ""
}
// louder is the setting that alerts more: page, then the default or warn,
// then off.
func louder(a, b string) string {
rank := func(s string) int {
switch s {
case report.AlertPage:
return 2
case report.AlertOff:
return 0
}
return 1
}
if rank(b) > rank(a) {
return b
}
return a
}
// adoptNamespace is a namespace as the adopt page lists it.
type adoptNamespace struct {
Name string `json:"name"`
Controllers int `json:"controllers"`
VMs int `json:"vms"`
Problems int `json:"problems"`
Alert string `json:"alert"` // set in Cairn OBS; "" for none
}
// adoptWorkload is a controller or VM as the adopt page lists it.
type adoptWorkload struct {
Name string `json:"name"` // namespace/kind/name
Kind string `json:"kind"`
Namespace string `json:"namespace"`
Label string `json:"label"` // "deployment/web", "vm/db1"
State string `json:"state"`
Word string `json:"word"`
Severity string `json:"severity"`
Annotated string `json:"annotated"` // from the cluster
Set *ExpectedWorkload `json:"set"` // in Cairn OBS, or null
Effective string `json:"effective"` // page, warn or off: what's in force
}
func (srv *Server) clusterExpected() *Expected {
if srv.expectedSource() == sourceFile {
return srv.currentExpected()
}
srv.expMu.Lock()
defer srv.expMu.Unlock()
return srv.expected
}
func effective(a string) string {
if a == "" {
return report.AlertWarn
}
return a
}
// handleClusterAdopt lists one cluster's namespaces, and its controllers
// and VMs a page at a time (?ns= and ?q= narrow them), with what's adopted.
func (srv *Server) handleClusterAdopt(w http.ResponseWriter, r *http.Request) {
cluster := r.PathValue("cluster")
if !slices.Contains(srv.clusterAgents(), cluster) {
writeJSON(w, http.StatusNotFound, map[string]string{"error": "No cluster called " + cluster + "."})
return
}
qv := r.URL.Query()
nsWant, q := qv.Get("ns"), strings.ToLower(strings.TrimSpace(qv.Get("q")))
offset, limit := page(r, 50, 500)
ec := srv.clusterExpected().Clusters[cluster]
ad := adoptionFor(&Expected{Clusters: map[string]ExpectedCluster{cluster: ec}}, cluster)
nss := map[string]*adoptNamespace{}
for n, a := range ec.Namespaces {
nss[n] = &adoptNamespace{Name: n, Alert: a}
}
rows := []adoptWorkload{}
present := map[string]bool{}
listed := false
for _, s := range srv.workloadSnapshot() {
if s.name != cluster {
continue
}
listed = !s.listed.IsZero()
ad.platform = isOpenShift(s.recs)
for _, rec := range s.recs {
if rec.Namespace == "" {
continue
}
n := nss[rec.Namespace]
if n == nil {
n = &adoptNamespace{Name: rec.Namespace}
nss[rec.Namespace] = n
}
if rec.Kind != report.KindController && rec.Kind != report.KindVM {
continue
}
key := recKey(rec)
present[key] = true
sev := stateSeverity(rec.State)
if rec.Kind == report.KindVM {
n.VMs++
} else {
n.Controllers++
}
if sev == "critical" || sev == "warning" {
n.Problems++
}
if (nsWant != "" && rec.Namespace != nsWant) || (q != "" && !strings.Contains(strings.ToLower(key), q)) {
continue
}
row := adoptWorkload{Name: key, Kind: rec.Kind, Namespace: rec.Namespace, Label: strings.TrimPrefix(key, rec.Namespace+"/"),
State: rec.State, Word: rec.Word, Severity: sev, Annotated: rec.Alert, Effective: effective(ad.alertKey(rec.Namespace, key, rec.Alert))}
if x, ok := ad.wl[key]; ok {
row.Set = &x
}
rows = append(rows, row)
}
}
// Adopted first, then problems, then by name.
slices.SortFunc(rows, func(a, b adoptWorkload) int {
return cmp.Or(cmp.Compare(boolRank(a.Set == nil), boolRank(b.Set == nil)), cmp.Compare(rank(a.Severity), rank(b.Severity)), strings.Compare(a.Name, b.Name))
})
namespaces := slices.SortedFunc(maps.Values(nss), func(a, b *adoptNamespace) int { return strings.Compare(a.Name, b.Name) })
missing := []ExpectedWorkload{}
if listed {
for _, x := range ec.Workloads {
if !present[x.Name] {
missing = append(missing, x)
}
}
}
writeJSON(w, http.StatusOK, map[string]any{"source": srv.expectedSource(), "file": srv.cfg.Expected, "cluster": cluster,
"namespaces": namespaces, "workloads": window(rows, offset, limit), "total": len(rows), "offset": offset, "limit": limit,
"missing": missing, "adopted": len(ec.Workloads)})
}
func boolRank(b bool) int {
if b {
return 1
}
return 0
}
// adoptForm changes one thing: a namespace, or a workload
// (namespace/kind/name). An empty alert takes it back to the default.
type adoptForm struct {
Namespace string `json:"namespace"`
Workload string `json:"workload"`
Alert string `json:"alert"`
Stopped bool `json:"stopped"`
}
func (srv *Server) handleClusterAdoptPut(w http.ResponseWriter, r *http.Request) {
if srv.readOnly(w, srv.expectedSource(), "adopted namespaces and workloads", srv.cfg.Expected) {
return
}
cluster := r.PathValue("cluster")
if !slices.Contains(srv.clusterAgents(), cluster) {
writeJSON(w, http.StatusNotFound, map[string]string{"error": "No cluster called " + cluster + "."})
return
}
var f adoptForm
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 16<<10)).Decode(&f); err != nil || (f.Namespace == "") == (f.Workload == "") {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "bad request: one namespace or one workload"})
return
}
srv.expMu.Lock()
defer srv.expMu.Unlock()
cur := srv.expected
if cur == nil {
cur = &Expected{}
}
old := cur.Clusters[cluster]
ec := ExpectedCluster{Namespaces: maps.Clone(old.Namespaces), Workloads: slices.Clone(old.Workloads), Subsystems: slices.Clone(old.Subsystems)}
what, did := f.Namespace, ""
if f.Namespace != "" {
if ec.Namespaces == nil {
ec.Namespaces = map[string]string{}
}
if f.Alert == "" {
delete(ec.Namespaces, f.Namespace)
did = "set back to the default"
} else {
ec.Namespaces[f.Namespace] = f.Alert
did = f.Alert
}
} else {
what = f.Workload
ec.Workloads = slices.DeleteFunc(ec.Workloads, func(x ExpectedWorkload) bool { return x.Name == f.Workload })
if f.Alert == "" {
did = "set back to the default"
} else {
ec.Workloads = append(ec.Workloads, ExpectedWorkload{Name: f.Workload, Alert: f.Alert, Stopped: f.Stopped})
slices.SortFunc(ec.Workloads, func(a, b ExpectedWorkload) int { return strings.Compare(a.Name, b.Name) })
did = f.Alert
if f.Stopped {
did += ", stopped on purpose"
}
}
}
if err := ec.check(); err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
return
}
next := &Expected{Hosts: cur.Hosts, CIJobs: cur.CIJobs, Clusters: maps.Clone(cur.Clusters)}
if next.Clusters == nil {
next.Clusters = map[string]ExpectedCluster{}
}
if len(ec.Namespaces) == 0 {
ec.Namespaces = nil
}
if len(ec.Workloads) == 0 {
ec.Workloads = nil
}
if ec.Namespaces == nil && ec.Workloads == nil && len(ec.Subsystems) == 0 {
delete(next.Clusters, cluster)
} else {
next.Clusters[cluster] = ec
}
if len(next.Clusters) == 0 {
next.Clusters = nil
}
if err := srv.putExpected(next, srv.who(r)); err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()})
return
}
srv.audit(srv.who(r), "set alerting in cluster "+cluster, fmt.Sprintf("%s: %s", what, did))
writeJSON(w, http.StatusOK, ec)
}