Files
T
jcoffey-dev db156c4f06
release / release (push) Successful in 7m6s
Cairn OBS
The new Cairn OBS: a self-hosted monitor for networks, hosts and
services. Licensed AGPLv3.
2026-11-14 03:27:09 -07:00

333 lines
12 KiB
Go

package clusterwatch
import (
"fmt"
"math"
"slices"
"strings"
"time"
"github.com/cairnobs/cairnobs/internal/report"
)
// KubeVirt VMs (docs/cluster-watcher.md, A VM is a VM, wherever it runs):
// one vm record for each VirtualMachine, with what its running instance
// (VMI), its newest live migration and its guest agent add. A VMI with no
// VirtualMachine is a VM of its own. The virt-launcher pod that runs a VMI
// isn't sent: the VM stands for it.
// vmParts is everything known about one VM beside the VirtualMachine.
type vmParts struct {
vmi *virtualMachineInstance
mig *vmiMigration // the newest migration of its VMI
fs *guestFS
}
func nsName(m meta) string { return m.Namespace + "/" + m.Name }
// vmRecords is a record for every VM and bare VMI.
func (b *builder) vmRecords(v view) []report.Workload {
vms := each[virtualMachine](v, "virtualmachines")
vmis := each[virtualMachineInstance](v, "virtualmachineinstances")
if len(vms) == 0 && len(vmis) == 0 {
return nil
}
parts := map[string]*vmParts{}
part := func(k string) *vmParts {
p := parts[k]
if p == nil {
p = &vmParts{}
parts[k] = p
}
return p
}
for _, i := range vmis {
part(nsName(i.Metadata)).vmi = i
}
for _, m := range each[vmiMigration](v, "virtualmachineinstancemigrations") {
p := part(m.Metadata.Namespace + "/" + m.Spec.VMIName)
if p.mig == nil || m.Metadata.CreationTimestamp.After(p.mig.Metadata.CreationTimestamp) ||
(m.Metadata.CreationTimestamp.Equal(p.mig.Metadata.CreationTimestamp) && m.Metadata.Name > p.mig.Metadata.Name) {
p.mig = m
}
}
for k, o := range v["guestfs"] {
if fs, ok := o.(*guestFS); ok {
part(k).fs = fs
}
}
var out []report.Workload
named := map[string]bool{}
for _, vm := range vms {
named[nsName(vm.Metadata)] = true
out = append(out, b.vm(vm, part(nsName(vm.Metadata))))
}
for _, i := range vmis {
if !named[nsName(i.Metadata)] {
out = append(out, b.bareVMI(i, part(nsName(i.Metadata))))
}
}
return out
}
// vmStates is KubeVirt's printable status in our states.
var vmStates = map[string]struct{ state, word string }{
"Running": {report.StateOK, "Running"},
"Stopped": {report.StateStopped, "Stopped"},
"Provisioning": {report.StateStarting, "Provisioning its volumes"},
"Starting": {report.StateStarting, "Starting"},
"WaitingForVolumeBinding": {report.StatePending, "Waiting for its volumes to bind"},
"WaitingForReceiver": {report.StatePending, "Waiting for a receiver"},
"Paused": {report.StateDegraded, "Paused"},
"Stopping": {report.StateStopped, "Stopping"},
"Terminating": {report.StateStopped, "Terminating"},
"Migrating": {report.StateUpdating, "Live migrating"},
"CrashLoopBackOff": {report.StateCrashloop, "Restarting (CrashLoopBackOff)"},
"ErrorUnschedulable": {report.StateDown, "Can't be scheduled (ErrorUnschedulable)"},
"ErrImagePull": {report.StateDown, "Can't pull its disk image (ErrImagePull)"},
"ImagePullBackOff": {report.StateDown, "Can't pull its disk image (ImagePullBackOff)"},
"ErrorPvcNotFound": {report.StateDown, "A volume it needs is missing (ErrorPvcNotFound)"},
"ErrorDataVolumeNotFound": {report.StateDown, "A DataVolume it needs is missing (ErrorDataVolumeNotFound)"},
"DataVolumeError": {report.StateDown, "A DataVolume it needs failed (DataVolumeError)"},
"Unknown": {report.StateUnknown, "Unknown"},
"": {report.StateUnknown, "No status yet"},
}
func (b *builder) vm(vm *virtualMachine, p *vmParts) report.Workload {
r := b.rec(report.KindVM, vm.Metadata.Name, vm.Metadata)
strategy := vm.strategy()
r.Facts = map[string]any{"run_strategy": strategy}
if u := vm.Spec.Template.Spec.Domain.Firmware.UUID; u != "" {
r.Facts["firmware_uuid"] = strings.ToLower(u)
}
st, ok := vmStates[vm.Status.PrintableStatus]
if !ok {
st = vmStates["Unknown"]
st.word = vm.Status.PrintableStatus
}
r.State, r.Word = st.state, st.word
r.Since = vm.Metadata.CreationTimestamp
if c := findCond(vm.Status.Conditions, "Ready"); c != nil && !c.LastTransitionTime.IsZero() {
r.Since = c.LastTransitionTime
}
// Meant to be running all the time, and isn't: KubeVirt should have
// started it again.
if r.State == report.StateStopped && strategy == "Always" && vm.Metadata.DeletionTimestamp == nil {
r.State, r.Word = report.StateDown, "Stopped, though its run strategy is Always"
}
if r.State == report.StateDown && vm.Status.PrintableStatus == "ErrorUnschedulable" {
if c := findCond(vm.Status.Conditions, "PodScheduled"); c != nil && c.Status == "False" && c.Message != "" {
// The scheduler's reason, without its note on preemption.
msg, _, _ := strings.Cut(c.Message, " preemption:")
r.Word += ": " + short(msg, 120)
}
}
b.vmFacts(&r, p)
return r
}
// bareVMI is a VMI nothing made: it's judged by its phase.
func (b *builder) bareVMI(i *virtualMachineInstance, p *vmParts) report.Workload {
r := b.rec(report.KindVM, i.Metadata.Name, i.Metadata)
r.Facts = map[string]any{"run_strategy": "None (a VMI with no VirtualMachine)"}
r.Since = i.Metadata.CreationTimestamp
switch i.Status.Phase {
case "Running":
r.State, r.Word = report.StateOK, "Running"
case "Pending", "Scheduling":
r.State, r.Word = report.StatePending, "Waiting to be scheduled"
case "Scheduled":
r.State, r.Word = report.StateStarting, "Starting"
case "Succeeded":
r.State, r.Word = report.StateStopped, "Shut down"
case "Failed":
r.State, r.Word = report.StateDown, "Failed"
if i.Status.Reason != "" {
r.Word += " (" + i.Status.Reason + ")"
}
default:
r.State, r.Word = report.StateUnknown, "Unknown"
}
if c := findCond(i.Status.Conditions, "Ready"); c != nil && !c.LastTransitionTime.IsZero() {
r.Since = c.LastTransitionTime
}
b.vmFacts(&r, p)
return r
}
// vmFacts adds what the running instance, its migrations and its guest
// agent say.
func (b *builder) vmFacts(r *report.Workload, p *vmParts) {
if i := p.vmi; i != nil {
r.Node = i.Status.NodeName
if u := i.Spec.Domain.Firmware.UUID; u != "" {
r.Facts["firmware_uuid"] = strings.ToLower(u)
}
if os := i.Status.GuestOSInfo.PrettyName; os != "" {
r.Facts["guest_os"] = os
}
var ips []string
for _, n := range i.Status.Interfaces {
for _, a := range append([]string{n.IPAddress}, n.IPAddresses...) {
// Link-local addresses say nothing anyone can reach.
if a != "" && !strings.HasPrefix(strings.ToLower(a), "fe80:") && !strings.HasPrefix(a, "169.254.") && !slices.Contains(ips, a) {
ips = append(ips, a)
}
}
}
if len(ips) > 4 {
ips = append(ips[:4], fmt.Sprintf("+%d more", len(ips)-4))
}
if len(ips) > 0 {
r.Facts["ips"] = strings.Join(ips, ", ")
}
if i.Status.Phase == "Running" {
r.Facts["agent_connected"] = i.agentConnected()
}
if m := i.Status.MigrationState; m != nil && !m.Completed && !m.Failed && m.TargetNode != "" && r.State == report.StateUpdating {
r.Word = "Live migrating to " + m.TargetNode
}
}
if m := p.mig; m != nil && m.Status.Phase == "Failed" {
at := m.Metadata.CreationTimestamp
for _, t := range m.Status.PhaseTransitionTimestamps {
if t.Phase == "Failed" {
at = t.Time
}
}
r.Facts["migration_failed"] = at.UTC().Format(time.RFC3339)
why := ""
for _, c := range m.Status.Conditions {
if c.Status == "True" && (c.Message != "" || c.Reason != "") {
why = c.Reason
if c.Message != "" {
why = c.Message
}
}
}
if why != "" {
r.Facts["migration_error"] = short(why, 160)
}
}
if fs := p.fs; fs != nil && p.vmi != nil && p.vmi.Status.Phase == "Running" {
var lines []string
worst, worstAt := -1, ""
// One file system mounted in several places (btrfs subvolumes, bind
// mounts) is said once, at its shortest mount point.
items := slices.Clone(fs.Items)
slices.SortStableFunc(items, func(a, b guestFSItem) int { return len(a.MountPoint) - len(b.MountPoint) })
type same struct {
typ string
used, total int64
}
seen := map[same]bool{}
for _, f := range items {
if f.TotalBytes <= 0 || f.MountPoint == "" || seen[same{f.FileSystemType, f.UsedBytes, f.TotalBytes}] {
continue
}
seen[same{f.FileSystemType, f.UsedBytes, f.TotalBytes}] = true
pct := int(math.Round(float64(f.UsedBytes) * 100 / float64(f.TotalBytes)))
lines = append(lines, fmt.Sprintf("%s %d%% of %s", f.MountPoint, pct, size(f.TotalBytes)))
if pct > worst || (pct == worst && f.MountPoint < worstAt) {
worst, worstAt = pct, f.MountPoint
}
}
if worst >= 0 {
slices.Sort(lines)
r.Facts["filesystems"] = strings.Join(lines, "; ")
r.Facts["fullest_mount"] = worstAt
r.Facts["fullest_pct"] = worst
}
}
}
// size is bytes for people, in the units guests show: "20 GiB".
func size(b int64) string {
const k = 1024
units := []string{"B", "KiB", "MiB", "GiB", "TiB", "PiB"}
f, u := float64(b), 0
for f >= k && u < len(units)-1 {
f /= k
u++
}
if f >= 10 || u == 0 {
return fmt.Sprintf("%.0f %s", f, units[u])
}
return fmt.Sprintf("%.1f %s", f, units[u])
}
// dvInProgress are DataVolume phases on the way to a filled volume.
var dvInProgress = map[string]string{
"ImportScheduled": "Import scheduled", "ImportInProgress": "Importing",
"CloneScheduled": "Clone scheduled", "CloneInProgress": "Cloning", "CSICloneInProgress": "Cloning",
"SnapshotForSmartCloneInProgress": "Cloning", "CloneFromSnapshotSourceInProgress": "Cloning",
"SmartClonePVCInProgress": "Cloning", "PrepClaimInProgress": "Preparing", "RebindInProgress": "Preparing",
"ExpansionInProgress": "Expanding", "NamespaceTransferInProgress": "Moving",
"UploadScheduled": "Upload scheduled", "UploadReady": "Waiting for an upload",
"PendingPopulation": "Waiting to be filled", "WaitForFirstConsumer": "Waiting for a VM to use it",
}
// dataVolume adds what CDI says about filling a volume to its PVC's
// record: an import or clone that's failing, or one under way (a volume
// being filled isn't Pending, it's busy).
func dataVolumeState(r *report.Workload, dv *dataVolume) {
st := dv.Status
if r.Facts == nil {
r.Facts = map[string]any{}
}
if st.Phase != "" {
r.Facts["datavolume"] = st.Phase
}
running := findCond(st.Conditions, "Running")
why := ""
if running != nil && running.Status == "False" && running.Message != "" {
why = ": " + short(running.Message, 140)
}
// CDI couldn't even make the claim (ErrClaimNotValid, …): no phase yet.
var claimErr *condition
for _, t := range []string{"Bound", "Ready"} {
if c := findCond(st.Conditions, t); c != nil && c.Status == "False" && strings.HasPrefix(c.Reason, "Err") {
claimErr = c
break
}
}
switch {
case claimErr != nil:
word := "Can't make its volume (" + claimErr.Reason + ")"
if claimErr.Message != "" {
word += ": " + short(claimErr.Message, 140)
}
r.State, r.Word, r.Since = report.StateDegraded, word, claimErr.LastTransitionTime
case st.Phase == "Failed":
r.State, r.Word = report.StateDegraded, "Import or clone failed"+why
if running != nil && !running.LastTransitionTime.IsZero() {
r.Since = running.LastTransitionTime
}
case running != nil && running.Status == "False" && running.Reason != "" && running.Reason != "Completed" && st.RestartCount > 0:
r.State, r.Word = report.StateDegraded, fmt.Sprintf("Import or clone failing, restarted %d times", st.RestartCount)+why
r.Since = running.LastTransitionTime
case dvInProgress[st.Phase] != "" && r.State != report.StateOK && r.State != report.StateDown:
word := dvInProgress[st.Phase]
if st.Progress != "" && st.Progress != "N/A" {
word += ", " + st.Progress
}
r.State, r.Word = report.StateStarting, word
}
if len(r.Facts) == 0 {
r.Facts = nil
}
}
// dvWorker is a pod CDI runs to fill a DataVolume's volume (importer,
// cloner, uploader): the volume's record says how that goes.
func dvWorker(p *pod, dvs map[string]*dataVolume) bool {
o := p.Metadata.controllerRef()
return o != nil && o.Kind == "PersistentVolumeClaim" && dvs[p.Metadata.Namespace+"/"+o.Name] != nil
}
// launcherPod is a virt-launcher pod: its VM stands for it.
func launcherPod(p *pod) bool {
o := p.Metadata.controllerRef()
return o != nil && o.Kind == "VirtualMachineInstance"
}