The reader adds SUBSYSTEM_INFO, SYSDISKSTAT, NETSTAT_INTERFACE_INFO, MESSAGE_QUEUE_INFO for QSYSOPR and JOB_QUEUE_INFO, still views only. Subsystems are parts, judged against subsystems: in expected.yaml by obs-beacond; disk units in trouble degrade their ASP; an interface set to start with TCP/IP that isn't active is degraded or down; QSYSOPR's unanswered inquiries (a job in MSGW) warn and its messages of severity 90 and up are critical; a held or orphaned job queue with jobs ready warns. Adopting a cluster workload keeps the subsystems list.
401 lines
15 KiB
Go
401 lines
15 KiB
Go
package ibmi
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/netip"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/cairnobs/cairnobs/internal/mapepire"
|
|
)
|
|
|
|
// The reads: views only, as Mapepire's basic query mode allows
|
|
// (docs/ibmi.md, decision 1), each named for what IBM i calls it.
|
|
const (
|
|
sqlStatus = `SELECT HOST_NAME, PARTITION_ID, PARTITION_NAME, NUMBER_OF_PARTITIONS, ELAPSED_TIME, ELAPSED_CPU_USED,
|
|
CONFIGURED_CPUS, CURRENT_CPU_CAPACITY, MAIN_STORAGE_SIZE, SYSTEM_ASP_STORAGE, SYSTEM_ASP_USED, CURRENT_TEMPORARY_STORAGE,
|
|
TOTAL_JOBS_IN_SYSTEM, MAXIMUM_JOBS_IN_SYSTEM, ACTIVE_JOBS_IN_SYSTEM, ACTIVE_THREADS_IN_SYSTEM, RESTRICTED_STATE,
|
|
MACHINE_TYPE, MACHINE_MODEL, SERIAL_NUMBER, ATTENTION_LIGHT FROM QSYS2.SYSTEM_STATUS_INFO_BASIC`
|
|
sqlRelease = `SELECT OS_VERSION, OS_RELEASE FROM SYSIBMADM.ENV_SYS_INFO`
|
|
// The Technology Refresh is the level of the TR group installed
|
|
// (SF99967 level 1 is 7.6 TR1). Its description is IBM's; a group
|
|
// that bundles the TR with others says more than that.
|
|
sqlTR = `SELECT PTF_GROUP_NAME, PTF_GROUP_DESCRIPTION, MAX(PTF_GROUP_LEVEL) AS LEVEL FROM QSYS2.GROUP_PTF_INFO
|
|
WHERE PTF_GROUP_DESCRIPTION LIKE '%TECHNOLOGY REFRESH%' AND PTF_GROUP_STATUS = 'INSTALLED'
|
|
GROUP BY PTF_GROUP_NAME, PTF_GROUP_DESCRIPTION`
|
|
sqlASPs = `SELECT ASP_NUMBER, DEVICE_DESCRIPTION_NAME, RESOURCE_NAME, ASP_STATE, ASP_TYPE, NUMBER_OF_DISK_UNITS, DISK_UNITS_PRESENT,
|
|
TOTAL_CAPACITY, TOTAL_CAPACITY_AVAILABLE, STORAGE_THRESHOLD_PERCENTAGE, OVERFLOW_STORAGE FROM QSYS2.ASP_INFO`
|
|
// Only the subsystems whose descriptions the profile may use are
|
|
// listed; IBM's own usually are public.
|
|
sqlSubsystems = `SELECT SUBSYSTEM_DESCRIPTION_LIBRARY, SUBSYSTEM_DESCRIPTION, STATUS, CURRENT_ACTIVE_JOBS, CONTROLLING_SUBSYSTEM,
|
|
TEXT_DESCRIPTION FROM QSYS2.SUBSYSTEM_INFO`
|
|
sqlDisks = `SELECT ASP_NUMBER, UNIT_NUMBER, RESOURCE_NAME, HARDWARE_STATUS, PROTECTION_STATUS, SSD_PFA_WARNING FROM QSYS2.SYSDISKSTAT`
|
|
sqlInterfaces = `SELECT CONNECTION_TYPE, INTERNET_ADDRESS, PREFIX_LENGTH, LINE_DESCRIPTION, INTERFACE_STATUS, AUTOSTART, INTERFACE_TEXT
|
|
FROM QSYS2.NETSTAT_INTERFACE_INFO`
|
|
// QSYSOPR's inquiries and replies of the last 30 days, and its
|
|
// messages of severity 80 and up of the last day. Keys are binary,
|
|
// which Mapepire would hand out as links, so they're read as hex; ages
|
|
// are the partition's own clock's.
|
|
sqlMessages = `SELECT HEX(MESSAGE_KEY) AS MSG_KEY, HEX(ASSOCIATED_MESSAGE_KEY) AS ASSOC_KEY, MESSAGE_ID, MESSAGE_TYPE, MESSAGE_TEXT,
|
|
SEVERITY, FROM_JOB, FROM_USER, TIMESTAMPDIFF(2, CHAR(NOW() - MESSAGE_TIMESTAMP)) AS AGE_SECONDS FROM QSYS2.MESSAGE_QUEUE_INFO
|
|
WHERE MESSAGE_QUEUE_LIBRARY = 'QSYS' AND MESSAGE_QUEUE_NAME = 'QSYSOPR' AND (
|
|
(MESSAGE_TYPE IN ('INQUIRY', 'REPLY') AND MESSAGE_TIMESTAMP > NOW() - 30 DAYS) OR (SEVERITY >= 80 AND MESSAGE_TIMESTAMP > NOW() - 1 DAY))
|
|
ORDER BY MESSAGE_TIMESTAMP DESC FETCH FIRST 4000 ROWS ONLY`
|
|
// Only the queues with jobs ready to run are read: one that's held,
|
|
// or that no active subsystem takes from, keeps them waiting.
|
|
sqlJobQueues = `SELECT JOB_QUEUE_LIBRARY, JOB_QUEUE_NAME, JOB_QUEUE_STATUS, RELEASED_JOBS, SUBSYSTEM_NAME FROM QSYS2.JOB_QUEUE_INFO
|
|
WHERE RELEASED_JOBS > 0`
|
|
)
|
|
|
|
// MaxMessages is how many unanswered inquiries, and how many severe
|
|
// messages, become records; the rest are counted on the partition's own.
|
|
const MaxMessages = 25
|
|
|
|
// System is SYSTEM_STATUS_INFO_BASIC's one row: the partition now.
|
|
type System struct {
|
|
HostName string
|
|
PartitionName string // as the HMC knows it; empty without *USE on QPMLPMGT
|
|
PartitionID int
|
|
Partitions int // on the server it runs on
|
|
MachineType string
|
|
MachineModel string
|
|
Serial string
|
|
CPUs int // virtual processors
|
|
Capacity float64 // processing units it may use now
|
|
MainStorageKB int64
|
|
SystemASPMB int64 // millions of bytes
|
|
SystemASPUsed float64
|
|
TempMB int64
|
|
Jobs int
|
|
MaxJobs int
|
|
ActiveJobs int
|
|
Threads int
|
|
Restricted bool
|
|
Attention bool // the control panel shows an attention SRC
|
|
}
|
|
|
|
// ASP is one auxiliary storage pool, from ASP_INFO.
|
|
type ASP struct {
|
|
Number int
|
|
Name string // an independent ASP's device description
|
|
State string // VARIED ON, VARIED OFF, ACTIVE, AVAILABLE, FAILED, NONE
|
|
Type string // SYSTEM, USER, PRIMARY, SECONDARY, UDFS
|
|
Units int
|
|
Present string // ALL, SOME, NONE
|
|
TotalMB int64 // 0 when it can't be told
|
|
FreeMB int64
|
|
Threshold int // % used at which IBM i warns QSYSOPR
|
|
OverflowMB int64
|
|
}
|
|
|
|
// Subsystem is one of SUBSYSTEM_INFO.
|
|
type Subsystem struct {
|
|
Library, Name string
|
|
Status string // ACTIVE, ENDING, INACTIVE, RESTRICTED, STARTING
|
|
Jobs int
|
|
Controlling bool
|
|
Text string
|
|
}
|
|
|
|
// Disk is one disk unit of SYSDISKSTAT that isn't fine: only those are
|
|
// kept, on their ASP's record.
|
|
type Disk struct {
|
|
ASP int
|
|
Unit int
|
|
Resource string // DMP012
|
|
Hardware string // HARDWARE_STATUS
|
|
Protection string // PROTECTION_STATUS
|
|
PFA bool // an SSD's predictive failure warning is logged
|
|
}
|
|
|
|
// Interface is one of NETSTAT_INTERFACE_INFO, loopback and link-local
|
|
// left out.
|
|
type Interface struct {
|
|
Address string
|
|
Prefix int
|
|
Line string // the line description: ETHLINE
|
|
Status string // ACTIVE, INACTIVE, STARTING, FAILED, RCYPND, ...
|
|
Autostart bool // started with TCP/IP: IBM i's own word that it should run
|
|
Text string
|
|
}
|
|
|
|
// Message is one of QSYSOPR's: an inquiry nobody has answered, or one of
|
|
// severity 80 and up.
|
|
type Message struct {
|
|
Key string // hex
|
|
ID string // CPA0701; empty for an impromptu message
|
|
Inquiry bool
|
|
Text string
|
|
Severity int
|
|
Job string // 123456/BATCH/NIGHTLY
|
|
User string
|
|
Age time.Duration
|
|
}
|
|
|
|
// JobQueue is a job queue whose released jobs can't start: it's held, or
|
|
// no active subsystem takes from it.
|
|
type JobQueue struct {
|
|
Library, Name string
|
|
Held bool
|
|
Waiting int // released jobs on it
|
|
Subsystem string // the active subsystem that takes from it; empty when none
|
|
}
|
|
|
|
// Snapshot is one read of a partition.
|
|
type Snapshot struct {
|
|
Name string // what it's kept under
|
|
System System
|
|
CPUPct *float64 // over the minute since the last read; nil on the first
|
|
Release string // 7.5
|
|
TR int // the Technology Refresh; 0 when it can't be told
|
|
ASPs []ASP
|
|
Disks []Disk // the disk units that aren't fine
|
|
Subsys []Subsystem
|
|
Ifaces []Interface
|
|
// Inquiries are QSYSOPR's unanswered ones, newest first, and Severe
|
|
// its messages of severity 80 and up of the last day; each list at
|
|
// most MaxMessages long, with the whole count beside it.
|
|
Inquiries []Message
|
|
Unanswered int
|
|
Severe []Message
|
|
SevereCount int
|
|
JobQueues []JobQueue
|
|
Problems map[string]string
|
|
At time.Time
|
|
}
|
|
|
|
// cpuMark is a read's ELAPSED_ columns: the CPU used on average since the
|
|
// connection's first read, and how long that is.
|
|
type cpuMark struct{ elapsed, used float64 }
|
|
|
|
// Read reads the partition once. Its status is what it's for: without it
|
|
// the read failed; anything else that can't be read is a problem on the
|
|
// partition's own record.
|
|
func (c *Client) Read(ctx context.Context, now time.Time) (*Snapshot, error) {
|
|
s := &Snapshot{Name: c.cfg.Name, Problems: map[string]string{}, At: now}
|
|
rows, err := c.query(ctx, sqlStatus)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(rows) == 0 {
|
|
return nil, fmt.Errorf("ibmi: SYSTEM_STATUS_INFO_BASIC gave no row")
|
|
}
|
|
r := rows[0]
|
|
s.System = System{
|
|
HostName: str(r, "HOST_NAME"), PartitionName: str(r, "PARTITION_NAME"), PartitionID: int(numOr0(r, "PARTITION_ID")),
|
|
Partitions: int(numOr0(r, "NUMBER_OF_PARTITIONS")), MachineType: str(r, "MACHINE_TYPE"), MachineModel: str(r, "MACHINE_MODEL"),
|
|
Serial: str(r, "SERIAL_NUMBER"), CPUs: int(numOr0(r, "CONFIGURED_CPUS")), Capacity: numOr0(r, "CURRENT_CPU_CAPACITY"),
|
|
MainStorageKB: int64(numOr0(r, "MAIN_STORAGE_SIZE")), SystemASPMB: int64(numOr0(r, "SYSTEM_ASP_STORAGE")),
|
|
SystemASPUsed: numOr0(r, "SYSTEM_ASP_USED"), TempMB: int64(numOr0(r, "CURRENT_TEMPORARY_STORAGE")),
|
|
Jobs: int(numOr0(r, "TOTAL_JOBS_IN_SYSTEM")), MaxJobs: int(numOr0(r, "MAXIMUM_JOBS_IN_SYSTEM")),
|
|
ActiveJobs: int(numOr0(r, "ACTIVE_JOBS_IN_SYSTEM")), Threads: int(numOr0(r, "ACTIVE_THREADS_IN_SYSTEM")),
|
|
Restricted: str(r, "RESTRICTED_STATE") == "YES", Attention: str(r, "ATTENTION_LIGHT") == "ON",
|
|
}
|
|
e, eok := num(r, "ELAPSED_TIME")
|
|
u, uok := num(r, "ELAPSED_CPU_USED")
|
|
if eok && uok {
|
|
mark := &cpuMark{e, u}
|
|
s.CPUPct = cpuBetween(c.last, mark)
|
|
c.last = mark
|
|
}
|
|
|
|
if rows, err := c.query(ctx, sqlRelease); err != nil {
|
|
s.Problems["release"] = err.Error()
|
|
} else if len(rows) > 0 {
|
|
v, rel := str(rows[0], "OS_VERSION"), str(rows[0], "OS_RELEASE")
|
|
if v != "" && rel != "" {
|
|
s.Release = v + "." + rel
|
|
}
|
|
}
|
|
// Reading the TR needs *USE on WRKPTFGRP, which a shop may not give:
|
|
// not knowing it isn't trouble with the partition.
|
|
if rows, err := c.query(ctx, sqlTR); err == nil {
|
|
s.TR = pickTR(rows)
|
|
}
|
|
|
|
if rows, err := c.query(ctx, sqlASPs); err != nil {
|
|
s.Problems["ASPs"] = err.Error()
|
|
} else {
|
|
for _, r := range rows {
|
|
a := ASP{Number: int(numOr0(r, "ASP_NUMBER")), Name: str(r, "DEVICE_DESCRIPTION_NAME"), State: str(r, "ASP_STATE"),
|
|
Type: str(r, "ASP_TYPE"), Units: int(numOr0(r, "NUMBER_OF_DISK_UNITS")), Present: str(r, "DISK_UNITS_PRESENT"),
|
|
Threshold: int(numOr0(r, "STORAGE_THRESHOLD_PERCENTAGE")), OverflowMB: int64(numOr0(r, "OVERFLOW_STORAGE"))}
|
|
if a.Name == "" {
|
|
a.Name = str(r, "RESOURCE_NAME")
|
|
}
|
|
// -2 is "too big to say"; null, "can't be told".
|
|
if t, ok := num(r, "TOTAL_CAPACITY"); ok && t > 0 {
|
|
if f, ok := num(r, "TOTAL_CAPACITY_AVAILABLE"); ok && f >= 0 {
|
|
a.TotalMB, a.FreeMB = int64(t), int64(f)
|
|
}
|
|
}
|
|
s.ASPs = append(s.ASPs, a)
|
|
}
|
|
sort.Slice(s.ASPs, func(i, j int) bool { return s.ASPs[i].Number < s.ASPs[j].Number })
|
|
}
|
|
if rows, err := c.query(ctx, sqlDisks); err != nil {
|
|
s.Problems["disk units"] = err.Error()
|
|
} else {
|
|
for _, r := range rows {
|
|
d := Disk{ASP: int(numOr0(r, "ASP_NUMBER")), Unit: int(numOr0(r, "UNIT_NUMBER")), Resource: str(r, "RESOURCE_NAME"),
|
|
Hardware: str(r, "HARDWARE_STATUS"), Protection: str(r, "PROTECTION_STATUS"), PFA: str(r, "SSD_PFA_WARNING") == "YES"}
|
|
if diskTrouble(d) != "" {
|
|
s.Disks = append(s.Disks, d)
|
|
}
|
|
}
|
|
sort.Slice(s.Disks, func(i, j int) bool { return s.Disks[i].Unit < s.Disks[j].Unit })
|
|
}
|
|
if rows, err := c.query(ctx, sqlSubsystems); err != nil {
|
|
s.Problems["subsystems"] = err.Error()
|
|
} else {
|
|
for _, r := range rows {
|
|
s.Subsys = append(s.Subsys, Subsystem{Library: str(r, "SUBSYSTEM_DESCRIPTION_LIBRARY"), Name: str(r, "SUBSYSTEM_DESCRIPTION"),
|
|
Status: str(r, "STATUS"), Jobs: int(numOr0(r, "CURRENT_ACTIVE_JOBS")), Controlling: str(r, "CONTROLLING_SUBSYSTEM") == "YES",
|
|
Text: str(r, "TEXT_DESCRIPTION")})
|
|
}
|
|
sort.Slice(s.Subsys, func(i, j int) bool {
|
|
return s.Subsys[i].Name+s.Subsys[i].Library < s.Subsys[j].Name+s.Subsys[j].Library
|
|
})
|
|
}
|
|
if rows, err := c.query(ctx, sqlInterfaces); err != nil {
|
|
s.Problems["interfaces"] = err.Error()
|
|
} else {
|
|
for _, r := range rows {
|
|
i := Interface{Address: str(r, "INTERNET_ADDRESS"), Prefix: int(numOr0(r, "PREFIX_LENGTH")), Line: str(r, "LINE_DESCRIPTION"),
|
|
Status: str(r, "INTERFACE_STATUS"), Autostart: str(r, "AUTOSTART") == "YES", Text: str(r, "INTERFACE_TEXT")}
|
|
if a, err := netip.ParseAddr(i.Address); err == nil && (a.IsLoopback() || a.IsLinkLocalUnicast()) {
|
|
continue
|
|
}
|
|
s.Ifaces = append(s.Ifaces, i)
|
|
}
|
|
sort.Slice(s.Ifaces, func(i, j int) bool { return s.Ifaces[i].Address < s.Ifaces[j].Address })
|
|
}
|
|
if rows, err := c.query(ctx, sqlMessages); err != nil {
|
|
s.Problems["QSYSOPR"] = err.Error()
|
|
} else {
|
|
readMessages(s, rows)
|
|
}
|
|
if rows, err := c.query(ctx, sqlJobQueues); err != nil {
|
|
s.Problems["job queues"] = err.Error()
|
|
} else {
|
|
for _, r := range rows {
|
|
q := JobQueue{Library: str(r, "JOB_QUEUE_LIBRARY"), Name: str(r, "JOB_QUEUE_NAME"), Held: str(r, "JOB_QUEUE_STATUS") == "HELD",
|
|
Waiting: int(numOr0(r, "RELEASED_JOBS")), Subsystem: str(r, "SUBSYSTEM_NAME")}
|
|
if q.Held || q.Subsystem == "" {
|
|
s.JobQueues = append(s.JobQueues, q)
|
|
}
|
|
}
|
|
sort.Slice(s.JobQueues, func(i, j int) bool {
|
|
return s.JobQueues[i].Name+s.JobQueues[i].Library < s.JobQueues[j].Name+s.JobQueues[j].Library
|
|
})
|
|
}
|
|
return s, nil
|
|
}
|
|
|
|
// readMessages sorts QSYSOPR's rows: an inquiry is answered when a reply
|
|
// names its key.
|
|
func readMessages(s *Snapshot, rows []mapepire.Row) {
|
|
answered := map[string]bool{}
|
|
for _, r := range rows {
|
|
if str(r, "MESSAGE_TYPE") == "REPLY" {
|
|
answered[str(r, "ASSOC_KEY")] = true
|
|
}
|
|
}
|
|
for _, r := range rows { // newest first
|
|
m := Message{Key: str(r, "MSG_KEY"), ID: str(r, "MESSAGE_ID"), Text: str(r, "MESSAGE_TEXT"), Severity: int(numOr0(r, "SEVERITY")),
|
|
Job: str(r, "FROM_JOB"), User: str(r, "FROM_USER"), Age: time.Duration(numOr0(r, "AGE_SECONDS")) * time.Second}
|
|
switch str(r, "MESSAGE_TYPE") {
|
|
case "REPLY":
|
|
case "INQUIRY":
|
|
if answered[m.Key] {
|
|
continue
|
|
}
|
|
m.Inquiry = true
|
|
s.Unanswered++
|
|
if len(s.Inquiries) < MaxMessages {
|
|
s.Inquiries = append(s.Inquiries, m)
|
|
}
|
|
default:
|
|
if m.Severity < 80 || m.Age > 24*time.Hour {
|
|
continue
|
|
}
|
|
s.SevereCount++
|
|
if len(s.Severe) < MaxMessages {
|
|
s.Severe = append(s.Severe, m)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// diskTrouble is what's wrong with a disk unit, "" when nothing: its
|
|
// hardware's status, then its protection's (device parity or mirroring),
|
|
// then an SSD's own warning that it will fail.
|
|
func diskTrouble(d Disk) string {
|
|
switch d.Hardware {
|
|
case "", "ACTIVE", "BUSY":
|
|
case "FAILED":
|
|
return "failed"
|
|
case "DEGRADED":
|
|
return "degraded (a component failed)"
|
|
case "HARDWARE FAILURE":
|
|
return "has a failed redundant component"
|
|
case "NOT READY":
|
|
return "isn't ready"
|
|
case "PARITY REBUILD":
|
|
return "has its parity being rebuilt"
|
|
case "POWER LOSS":
|
|
return "has lost power"
|
|
case "READ WRITE PROTECTED", "WRITE PROTECTED":
|
|
return "is " + strings.ToLower(d.Hardware)
|
|
case "UNPROTECTED":
|
|
return "is unprotected: its parity set lost a unit"
|
|
default:
|
|
return "is " + strings.ToLower(d.Hardware)
|
|
}
|
|
switch d.Protection {
|
|
case "", "ACTIVE", "BUSY":
|
|
case "SUSPEND":
|
|
return "has its mirroring suspended"
|
|
case "RESUME", "RESUME PENDING":
|
|
return "has its mirroring resuming"
|
|
default:
|
|
return "protection " + strings.ToLower(d.Protection)
|
|
}
|
|
if d.PFA {
|
|
return "predicts its own failure (SSD)"
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// cpuBetween is the CPU used between two reads of the same connection:
|
|
// each averages since its first read, so the minute between is what the
|
|
// later one adds. A new connection (elapsed went back) gives nothing yet.
|
|
func cpuBetween(prev, cur *cpuMark) *float64 {
|
|
if prev == nil || cur.elapsed <= prev.elapsed {
|
|
return nil
|
|
}
|
|
p := (cur.used*cur.elapsed - prev.used*prev.elapsed) / (cur.elapsed - prev.elapsed)
|
|
p = round1(min(max(p, 0), 100))
|
|
return &p
|
|
}
|
|
|
|
// pickTR is the Technology Refresh from the TR groups installed: the
|
|
// group that's only the TR when there is one, its highest level.
|
|
func pickTR(rows []mapepire.Row) int {
|
|
best, bestPure := 0, false
|
|
for _, r := range rows {
|
|
d := strings.ToUpper(str(r, "PTF_GROUP_DESCRIPTION"))
|
|
pure := strings.TrimSpace(d) == "TECHNOLOGY REFRESH"
|
|
lvl := int(numOr0(r, "LEVEL"))
|
|
switch {
|
|
case pure && !bestPure:
|
|
best, bestPure = lvl, true
|
|
case pure == bestPure && lvl > best:
|
|
best = lvl
|
|
}
|
|
}
|
|
return best
|
|
}
|