Files
cairnobs/internal/server/expected.go
T
jcoffey-dev f71f968c7f
ci / postgres (pull_request) Successful in 3m35s
ci / check (pull_request) Successful in 7m27s
IBM i, steps 2 and 3: subsystems, disks, interfaces, QSYSOPR and job queues
The reader adds SUBSYSTEM_INFO, SYSDISKSTAT, NETSTAT_INTERFACE_INFO,
MESSAGE_QUEUE_INFO for QSYSOPR and JOB_QUEUE_INFO, still views only.
Subsystems are parts, judged against subsystems: in expected.yaml by
obs-beacond; disk units in trouble degrade their ASP; an interface set
to start with TCP/IP that isn't active is degraded or down; QSYSOPR's
unanswered inquiries (a job in MSGW) warn and its messages of severity
90 and up are critical; a held or orphaned job queue with jobs ready
warns. Adopting a cluster workload keeps the subsystems list.
2026-11-14 23:51:03 -07:00

318 lines
9.2 KiB
Go

package server
import (
"bytes"
"errors"
"fmt"
"io"
"os"
"path"
"regexp"
"gopkg.in/yaml.v3"
"github.com/cairnobs/cairnobs/internal/report"
)
// Expected is what should run where. See docs/formats.md.
type Expected struct {
Hosts map[string]ExpectedHost `yaml:"hosts"`
CIJobs []string `yaml:"ci_jobs"` // name patterns: counted, never listed or alerted on
Clusters map[string]ExpectedCluster `yaml:"clusters,omitempty"`
}
// ExpectedCluster is what's adopted in one cluster (docs/cluster-watcher.md,
// What pages, and who decides): whole namespaces, and single workloads.
// Adopted ones page; the rest only warn on the pages.
type ExpectedCluster struct {
Namespaces map[string]string `yaml:"namespaces,omitempty"` // namespace → page, warn or off
Workloads []ExpectedWorkload `yaml:"workloads,omitempty"`
// Subsystems are an IBM i partition's that must be active
// (docs/ibmi.md, step 2): QINTER, or ORDLIB/ORDERS when two libraries
// hold one of that name.
Subsystems []string `yaml:"subsystems,omitempty"`
}
// subsystemName is an expected subsystem: an IBM i name, or
// library/name.
var subsystemName = regexp.MustCompile(`^(?i)([a-z$#@][a-z0-9$#@_.]{0,9}/)?[a-z$#@][a-z0-9$#@_.]{0,9}$`)
// ExpectedWorkload is one adopted workload: "shop/deployment/web", or
// "vms/vm/db1" for a VM. Alert is page (the default), warn or off.
// Stopped marks a VM that's stopped on purpose; otherwise an adopted VM
// should be running.
type ExpectedWorkload struct {
Name string `yaml:"name" json:"name"`
Alert string `yaml:"alert,omitempty" json:"alert"`
Stopped bool `yaml:"stopped,omitempty" json:"stopped"`
}
// k8sName is a Kubernetes namespace or object name, and workloadKey an
// adopted workload's namespace/kind/name.
var (
k8sName = regexp.MustCompile(`^[a-z0-9]([a-z0-9.-]{0,251}[a-z0-9])?$`)
workloadKey = regexp.MustCompile(`^([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)/([a-z0-9]{1,63})/([a-z0-9]([a-z0-9.-]{0,251}[a-z0-9])?)$`)
)
func checkAlert(a string, empty bool) error {
switch a {
case report.AlertPage, report.AlertWarn, report.AlertOff:
return nil
case "":
if empty {
return nil
}
}
return fmt.Errorf("alert %q isn't page, warn or off", a)
}
func (ec ExpectedCluster) check() error {
for _, b := range ec.Subsystems {
if !subsystemName.MatchString(b) {
return fmt.Errorf("subsystem %q isn't an IBM i name, or library/name", b)
}
}
for ns, a := range ec.Namespaces {
if !k8sName.MatchString(ns) || len(ns) > 63 {
return fmt.Errorf("%q isn't a namespace name", ns)
}
if err := checkAlert(a, false); err != nil {
return fmt.Errorf("namespace %s: %w", ns, err)
}
}
seen := map[string]bool{}
for _, w := range ec.Workloads {
m := workloadKey.FindStringSubmatch(w.Name)
if m == nil {
return fmt.Errorf("%q isn't namespace/kind/name, as in shop/deployment/web", w.Name)
}
if seen[w.Name] {
return fmt.Errorf("%s is listed twice", w.Name)
}
seen[w.Name] = true
if err := checkAlert(w.Alert, true); err != nil {
return fmt.Errorf("%s: %w", w.Name, err)
}
if w.Stopped && m[3] != "vm" {
return fmt.Errorf("%s: only a VM can be stopped on purpose", w.Name)
}
}
return nil
}
// ExpectedHost lists container name patterns for one host. "*" matches any
// run of characters, as in a shell.
type ExpectedHost struct {
Running []string `yaml:"running"`
Stopped []StoppedRule `yaml:"stopped"`
Units []string `yaml:"units"` // systemd units that must be active ("inbuxa" means inbuxa.service)
}
// StoppedRule marks containers that are stopped on purpose.
type StoppedRule struct {
Match string `yaml:"match" json:"match"`
Note string `yaml:"note" json:"note"`
}
// LoadExpected reads an expected-containers file. An empty path means no
// expectations: containers are judged on their own state only.
func LoadExpected(p string) (*Expected, error) {
e := &Expected{}
if p == "" {
return e, nil
}
b, err := os.ReadFile(p)
if err != nil {
return nil, err
}
if e, err = parseExpected(b); err != nil {
return nil, fmt.Errorf("%s: %w", p, err)
}
return e, nil
}
// parseExpected reads and checks an expected.yaml document.
func parseExpected(b []byte) (*Expected, error) {
e := &Expected{}
dec := yaml.NewDecoder(bytes.NewReader(b))
dec.KnownFields(true)
if err := dec.Decode(e); err != nil && !errors.Is(err, io.EOF) {
return nil, err
}
for _, pat := range e.CIJobs {
if _, err := path.Match(pat, ""); err != nil {
return nil, fmt.Errorf("ci_jobs: bad pattern %q", pat)
}
}
for h, eh := range e.Hosts {
if err := eh.check(); err != nil {
return nil, fmt.Errorf("host %s: %w", h, err)
}
}
for c, ec := range e.Clusters {
if err := ec.check(); err != nil {
return nil, fmt.Errorf("cluster %s: %w", c, err)
}
}
return e, nil
}
// unitName is what a unit in an expected list may be called.
var unitName = regexp.MustCompile(`^[A-Za-z0-9@._:\\-]{1,200}$`)
func (eh ExpectedHost) check() error {
for _, pat := range append(append([]string{}, eh.Running...), stoppedPatterns(eh.Stopped)...) {
if pat == "" {
return errors.New("an empty container name")
}
if _, err := path.Match(pat, ""); err != nil {
return fmt.Errorf("bad pattern %q", pat)
}
}
for _, u := range eh.Units {
if !unitName.MatchString(u) {
return fmt.Errorf("unit %q isn't a systemd unit name", u)
}
}
return nil
}
func stoppedPatterns(rs []StoppedRule) []string {
out := make([]string, len(rs))
for i, r := range rs {
out[i] = r.Match
}
return out
}
func matches(pats []string, name string) (string, bool) {
for _, p := range pats {
if ok, _ := path.Match(p, name); ok {
return p, true
}
}
return "", false
}
// Verdict is Cairn OBS's judgement of one container.
type Verdict struct {
Severity string `json:"severity"` // good, warning, critical, off
Word string `json:"word"`
Detail string `json:"detail,omitempty"`
}
// ContainerRow is a container as the UI shows it. Missing containers (expected
// but not there at all) have State "missing" and no ID.
type ContainerRow struct {
report.Container
Host string `json:"host"`
Expected bool `json:"expected"` // named in the host's running list
Verdict Verdict `json:"verdict"`
}
// ContainerSummary counts one host's containers, or all of them.
type ContainerSummary struct {
Running int `json:"running"` // up and fine
Problems int `json:"problems"` // warning or critical
Stopped int `json:"stopped"` // stopped on purpose
CI int `json:"ci"` // CI job containers right now
Total int `json:"total"` // listed rows
}
func (s *ContainerSummary) add(o ContainerSummary) {
s.Running += o.Running
s.Problems += o.Problems
s.Stopped += o.Stopped
s.CI += o.CI
s.Total += o.Total
}
// Judge turns one host's container list into rows with verdicts.
func (e *Expected) Judge(host string, list []report.Container) ([]ContainerRow, ContainerSummary) {
eh, haveHost := e.Hosts[host]
var rows []ContainerRow
var sum ContainerSummary
found := map[string]bool{} // running patterns that matched something
for _, c := range list {
if _, ci := matches(e.CIJobs, c.Name); ci {
if c.State == "running" {
sum.CI++
}
continue
}
row := ContainerRow{Container: c, Host: host}
if rule, ok := stoppedRule(eh.Stopped, c.Name); ok {
if c.State == "running" {
row.Verdict = Verdict{"warning", "Running", "listed as stopped on purpose: " + rule.Note}
} else {
row.Verdict = Verdict{"off", "Stopped on purpose", rule.Note}
}
} else if pat, ok := matches(eh.Running, c.Name); ok {
found[pat] = true
row.Expected = true
row.Verdict = stateVerdict(c, true)
} else {
row.Verdict = stateVerdict(c, false)
if haveHost && c.State == "running" && row.Verdict.Severity == "good" {
row.Verdict = Verdict{"warning", "Not expected", "running, but not in the expected list for " + host}
}
}
rows = append(rows, row)
}
for _, pat := range eh.Running {
if !found[pat] {
rows = append(rows, ContainerRow{
Container: report.Container{Name: pat, State: "missing"},
Host: host, Expected: true,
Verdict: Verdict{"critical", "Missing", "expected on " + host + ", but Docker doesn't list it"},
})
}
}
for _, r := range rows {
sum.Total++
switch r.Verdict.Severity {
case "good":
sum.Running++
case "off":
if r.Verdict.Word == "Stopped on purpose" {
sum.Stopped++
}
default:
sum.Problems++
}
}
return rows, sum
}
func stoppedRule(rs []StoppedRule, name string) (StoppedRule, bool) {
for _, r := range rs {
if ok, _ := path.Match(r.Match, name); ok {
return r, true
}
}
return StoppedRule{}, false
}
// stateVerdict judges a container on Docker's own state. expected says
// whether something should be running.
func stateVerdict(c report.Container, expected bool) Verdict {
switch c.State {
case "running":
switch c.Health {
case "unhealthy":
return Verdict{"critical", "Unhealthy", c.Status}
case "starting":
return Verdict{"warning", "Starting", c.Status}
}
return Verdict{"good", "Running", c.Status}
case "restarting":
return Verdict{"critical", "Restarting", c.Status}
case "paused":
return Verdict{"warning", "Paused", c.Status}
}
if expected {
return Verdict{"critical", "Not running", c.Status}
}
return Verdict{"off", "Stopped", c.Status}
}