release / release (push) Successful in 7m6s
The new Cairn OBS: a self-hosted monitor for networks, hosts and services. Licensed AGPLv3.
333 lines
12 KiB
Go
333 lines
12 KiB
Go
package clusterwatch
|
|
|
|
import (
|
|
"fmt"
|
|
"math"
|
|
"slices"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/cairnobs/cairnobs/internal/report"
|
|
)
|
|
|
|
// KubeVirt VMs (docs/cluster-watcher.md, A VM is a VM, wherever it runs):
|
|
// one vm record for each VirtualMachine, with what its running instance
|
|
// (VMI), its newest live migration and its guest agent add. A VMI with no
|
|
// VirtualMachine is a VM of its own. The virt-launcher pod that runs a VMI
|
|
// isn't sent: the VM stands for it.
|
|
|
|
// vmParts is everything known about one VM beside the VirtualMachine.
|
|
type vmParts struct {
|
|
vmi *virtualMachineInstance
|
|
mig *vmiMigration // the newest migration of its VMI
|
|
fs *guestFS
|
|
}
|
|
|
|
func nsName(m meta) string { return m.Namespace + "/" + m.Name }
|
|
|
|
// vmRecords is a record for every VM and bare VMI.
|
|
func (b *builder) vmRecords(v view) []report.Workload {
|
|
vms := each[virtualMachine](v, "virtualmachines")
|
|
vmis := each[virtualMachineInstance](v, "virtualmachineinstances")
|
|
if len(vms) == 0 && len(vmis) == 0 {
|
|
return nil
|
|
}
|
|
parts := map[string]*vmParts{}
|
|
part := func(k string) *vmParts {
|
|
p := parts[k]
|
|
if p == nil {
|
|
p = &vmParts{}
|
|
parts[k] = p
|
|
}
|
|
return p
|
|
}
|
|
for _, i := range vmis {
|
|
part(nsName(i.Metadata)).vmi = i
|
|
}
|
|
for _, m := range each[vmiMigration](v, "virtualmachineinstancemigrations") {
|
|
p := part(m.Metadata.Namespace + "/" + m.Spec.VMIName)
|
|
if p.mig == nil || m.Metadata.CreationTimestamp.After(p.mig.Metadata.CreationTimestamp) ||
|
|
(m.Metadata.CreationTimestamp.Equal(p.mig.Metadata.CreationTimestamp) && m.Metadata.Name > p.mig.Metadata.Name) {
|
|
p.mig = m
|
|
}
|
|
}
|
|
for k, o := range v["guestfs"] {
|
|
if fs, ok := o.(*guestFS); ok {
|
|
part(k).fs = fs
|
|
}
|
|
}
|
|
var out []report.Workload
|
|
named := map[string]bool{}
|
|
for _, vm := range vms {
|
|
named[nsName(vm.Metadata)] = true
|
|
out = append(out, b.vm(vm, part(nsName(vm.Metadata))))
|
|
}
|
|
for _, i := range vmis {
|
|
if !named[nsName(i.Metadata)] {
|
|
out = append(out, b.bareVMI(i, part(nsName(i.Metadata))))
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// vmStates is KubeVirt's printable status in our states.
|
|
var vmStates = map[string]struct{ state, word string }{
|
|
"Running": {report.StateOK, "Running"},
|
|
"Stopped": {report.StateStopped, "Stopped"},
|
|
"Provisioning": {report.StateStarting, "Provisioning its volumes"},
|
|
"Starting": {report.StateStarting, "Starting"},
|
|
"WaitingForVolumeBinding": {report.StatePending, "Waiting for its volumes to bind"},
|
|
"WaitingForReceiver": {report.StatePending, "Waiting for a receiver"},
|
|
"Paused": {report.StateDegraded, "Paused"},
|
|
"Stopping": {report.StateStopped, "Stopping"},
|
|
"Terminating": {report.StateStopped, "Terminating"},
|
|
"Migrating": {report.StateUpdating, "Live migrating"},
|
|
"CrashLoopBackOff": {report.StateCrashloop, "Restarting (CrashLoopBackOff)"},
|
|
"ErrorUnschedulable": {report.StateDown, "Can't be scheduled (ErrorUnschedulable)"},
|
|
"ErrImagePull": {report.StateDown, "Can't pull its disk image (ErrImagePull)"},
|
|
"ImagePullBackOff": {report.StateDown, "Can't pull its disk image (ImagePullBackOff)"},
|
|
"ErrorPvcNotFound": {report.StateDown, "A volume it needs is missing (ErrorPvcNotFound)"},
|
|
"ErrorDataVolumeNotFound": {report.StateDown, "A DataVolume it needs is missing (ErrorDataVolumeNotFound)"},
|
|
"DataVolumeError": {report.StateDown, "A DataVolume it needs failed (DataVolumeError)"},
|
|
"Unknown": {report.StateUnknown, "Unknown"},
|
|
"": {report.StateUnknown, "No status yet"},
|
|
}
|
|
|
|
func (b *builder) vm(vm *virtualMachine, p *vmParts) report.Workload {
|
|
r := b.rec(report.KindVM, vm.Metadata.Name, vm.Metadata)
|
|
strategy := vm.strategy()
|
|
r.Facts = map[string]any{"run_strategy": strategy}
|
|
if u := vm.Spec.Template.Spec.Domain.Firmware.UUID; u != "" {
|
|
r.Facts["firmware_uuid"] = strings.ToLower(u)
|
|
}
|
|
st, ok := vmStates[vm.Status.PrintableStatus]
|
|
if !ok {
|
|
st = vmStates["Unknown"]
|
|
st.word = vm.Status.PrintableStatus
|
|
}
|
|
r.State, r.Word = st.state, st.word
|
|
r.Since = vm.Metadata.CreationTimestamp
|
|
if c := findCond(vm.Status.Conditions, "Ready"); c != nil && !c.LastTransitionTime.IsZero() {
|
|
r.Since = c.LastTransitionTime
|
|
}
|
|
// Meant to be running all the time, and isn't: KubeVirt should have
|
|
// started it again.
|
|
if r.State == report.StateStopped && strategy == "Always" && vm.Metadata.DeletionTimestamp == nil {
|
|
r.State, r.Word = report.StateDown, "Stopped, though its run strategy is Always"
|
|
}
|
|
if r.State == report.StateDown && vm.Status.PrintableStatus == "ErrorUnschedulable" {
|
|
if c := findCond(vm.Status.Conditions, "PodScheduled"); c != nil && c.Status == "False" && c.Message != "" {
|
|
// The scheduler's reason, without its note on preemption.
|
|
msg, _, _ := strings.Cut(c.Message, " preemption:")
|
|
r.Word += ": " + short(msg, 120)
|
|
}
|
|
}
|
|
b.vmFacts(&r, p)
|
|
return r
|
|
}
|
|
|
|
// bareVMI is a VMI nothing made: it's judged by its phase.
|
|
func (b *builder) bareVMI(i *virtualMachineInstance, p *vmParts) report.Workload {
|
|
r := b.rec(report.KindVM, i.Metadata.Name, i.Metadata)
|
|
r.Facts = map[string]any{"run_strategy": "None (a VMI with no VirtualMachine)"}
|
|
r.Since = i.Metadata.CreationTimestamp
|
|
switch i.Status.Phase {
|
|
case "Running":
|
|
r.State, r.Word = report.StateOK, "Running"
|
|
case "Pending", "Scheduling":
|
|
r.State, r.Word = report.StatePending, "Waiting to be scheduled"
|
|
case "Scheduled":
|
|
r.State, r.Word = report.StateStarting, "Starting"
|
|
case "Succeeded":
|
|
r.State, r.Word = report.StateStopped, "Shut down"
|
|
case "Failed":
|
|
r.State, r.Word = report.StateDown, "Failed"
|
|
if i.Status.Reason != "" {
|
|
r.Word += " (" + i.Status.Reason + ")"
|
|
}
|
|
default:
|
|
r.State, r.Word = report.StateUnknown, "Unknown"
|
|
}
|
|
if c := findCond(i.Status.Conditions, "Ready"); c != nil && !c.LastTransitionTime.IsZero() {
|
|
r.Since = c.LastTransitionTime
|
|
}
|
|
b.vmFacts(&r, p)
|
|
return r
|
|
}
|
|
|
|
// vmFacts adds what the running instance, its migrations and its guest
|
|
// agent say.
|
|
func (b *builder) vmFacts(r *report.Workload, p *vmParts) {
|
|
if i := p.vmi; i != nil {
|
|
r.Node = i.Status.NodeName
|
|
if u := i.Spec.Domain.Firmware.UUID; u != "" {
|
|
r.Facts["firmware_uuid"] = strings.ToLower(u)
|
|
}
|
|
if os := i.Status.GuestOSInfo.PrettyName; os != "" {
|
|
r.Facts["guest_os"] = os
|
|
}
|
|
var ips []string
|
|
for _, n := range i.Status.Interfaces {
|
|
for _, a := range append([]string{n.IPAddress}, n.IPAddresses...) {
|
|
// Link-local addresses say nothing anyone can reach.
|
|
if a != "" && !strings.HasPrefix(strings.ToLower(a), "fe80:") && !strings.HasPrefix(a, "169.254.") && !slices.Contains(ips, a) {
|
|
ips = append(ips, a)
|
|
}
|
|
}
|
|
}
|
|
if len(ips) > 4 {
|
|
ips = append(ips[:4], fmt.Sprintf("+%d more", len(ips)-4))
|
|
}
|
|
if len(ips) > 0 {
|
|
r.Facts["ips"] = strings.Join(ips, ", ")
|
|
}
|
|
if i.Status.Phase == "Running" {
|
|
r.Facts["agent_connected"] = i.agentConnected()
|
|
}
|
|
if m := i.Status.MigrationState; m != nil && !m.Completed && !m.Failed && m.TargetNode != "" && r.State == report.StateUpdating {
|
|
r.Word = "Live migrating to " + m.TargetNode
|
|
}
|
|
}
|
|
if m := p.mig; m != nil && m.Status.Phase == "Failed" {
|
|
at := m.Metadata.CreationTimestamp
|
|
for _, t := range m.Status.PhaseTransitionTimestamps {
|
|
if t.Phase == "Failed" {
|
|
at = t.Time
|
|
}
|
|
}
|
|
r.Facts["migration_failed"] = at.UTC().Format(time.RFC3339)
|
|
why := ""
|
|
for _, c := range m.Status.Conditions {
|
|
if c.Status == "True" && (c.Message != "" || c.Reason != "") {
|
|
why = c.Reason
|
|
if c.Message != "" {
|
|
why = c.Message
|
|
}
|
|
}
|
|
}
|
|
if why != "" {
|
|
r.Facts["migration_error"] = short(why, 160)
|
|
}
|
|
}
|
|
if fs := p.fs; fs != nil && p.vmi != nil && p.vmi.Status.Phase == "Running" {
|
|
var lines []string
|
|
worst, worstAt := -1, ""
|
|
// One file system mounted in several places (btrfs subvolumes, bind
|
|
// mounts) is said once, at its shortest mount point.
|
|
items := slices.Clone(fs.Items)
|
|
slices.SortStableFunc(items, func(a, b guestFSItem) int { return len(a.MountPoint) - len(b.MountPoint) })
|
|
type same struct {
|
|
typ string
|
|
used, total int64
|
|
}
|
|
seen := map[same]bool{}
|
|
for _, f := range items {
|
|
if f.TotalBytes <= 0 || f.MountPoint == "" || seen[same{f.FileSystemType, f.UsedBytes, f.TotalBytes}] {
|
|
continue
|
|
}
|
|
seen[same{f.FileSystemType, f.UsedBytes, f.TotalBytes}] = true
|
|
pct := int(math.Round(float64(f.UsedBytes) * 100 / float64(f.TotalBytes)))
|
|
lines = append(lines, fmt.Sprintf("%s %d%% of %s", f.MountPoint, pct, size(f.TotalBytes)))
|
|
if pct > worst || (pct == worst && f.MountPoint < worstAt) {
|
|
worst, worstAt = pct, f.MountPoint
|
|
}
|
|
}
|
|
if worst >= 0 {
|
|
slices.Sort(lines)
|
|
r.Facts["filesystems"] = strings.Join(lines, "; ")
|
|
r.Facts["fullest_mount"] = worstAt
|
|
r.Facts["fullest_pct"] = worst
|
|
}
|
|
}
|
|
}
|
|
|
|
// size is bytes for people, in the units guests show: "20 GiB".
|
|
func size(b int64) string {
|
|
const k = 1024
|
|
units := []string{"B", "KiB", "MiB", "GiB", "TiB", "PiB"}
|
|
f, u := float64(b), 0
|
|
for f >= k && u < len(units)-1 {
|
|
f /= k
|
|
u++
|
|
}
|
|
if f >= 10 || u == 0 {
|
|
return fmt.Sprintf("%.0f %s", f, units[u])
|
|
}
|
|
return fmt.Sprintf("%.1f %s", f, units[u])
|
|
}
|
|
|
|
// dvInProgress are DataVolume phases on the way to a filled volume.
|
|
var dvInProgress = map[string]string{
|
|
"ImportScheduled": "Import scheduled", "ImportInProgress": "Importing",
|
|
"CloneScheduled": "Clone scheduled", "CloneInProgress": "Cloning", "CSICloneInProgress": "Cloning",
|
|
"SnapshotForSmartCloneInProgress": "Cloning", "CloneFromSnapshotSourceInProgress": "Cloning",
|
|
"SmartClonePVCInProgress": "Cloning", "PrepClaimInProgress": "Preparing", "RebindInProgress": "Preparing",
|
|
"ExpansionInProgress": "Expanding", "NamespaceTransferInProgress": "Moving",
|
|
"UploadScheduled": "Upload scheduled", "UploadReady": "Waiting for an upload",
|
|
"PendingPopulation": "Waiting to be filled", "WaitForFirstConsumer": "Waiting for a VM to use it",
|
|
}
|
|
|
|
// dataVolume adds what CDI says about filling a volume to its PVC's
|
|
// record: an import or clone that's failing, or one under way (a volume
|
|
// being filled isn't Pending, it's busy).
|
|
func dataVolumeState(r *report.Workload, dv *dataVolume) {
|
|
st := dv.Status
|
|
if r.Facts == nil {
|
|
r.Facts = map[string]any{}
|
|
}
|
|
if st.Phase != "" {
|
|
r.Facts["datavolume"] = st.Phase
|
|
}
|
|
running := findCond(st.Conditions, "Running")
|
|
why := ""
|
|
if running != nil && running.Status == "False" && running.Message != "" {
|
|
why = ": " + short(running.Message, 140)
|
|
}
|
|
// CDI couldn't even make the claim (ErrClaimNotValid, …): no phase yet.
|
|
var claimErr *condition
|
|
for _, t := range []string{"Bound", "Ready"} {
|
|
if c := findCond(st.Conditions, t); c != nil && c.Status == "False" && strings.HasPrefix(c.Reason, "Err") {
|
|
claimErr = c
|
|
break
|
|
}
|
|
}
|
|
switch {
|
|
case claimErr != nil:
|
|
word := "Can't make its volume (" + claimErr.Reason + ")"
|
|
if claimErr.Message != "" {
|
|
word += ": " + short(claimErr.Message, 140)
|
|
}
|
|
r.State, r.Word, r.Since = report.StateDegraded, word, claimErr.LastTransitionTime
|
|
case st.Phase == "Failed":
|
|
r.State, r.Word = report.StateDegraded, "Import or clone failed"+why
|
|
if running != nil && !running.LastTransitionTime.IsZero() {
|
|
r.Since = running.LastTransitionTime
|
|
}
|
|
case running != nil && running.Status == "False" && running.Reason != "" && running.Reason != "Completed" && st.RestartCount > 0:
|
|
r.State, r.Word = report.StateDegraded, fmt.Sprintf("Import or clone failing, restarted %d times", st.RestartCount)+why
|
|
r.Since = running.LastTransitionTime
|
|
case dvInProgress[st.Phase] != "" && r.State != report.StateOK && r.State != report.StateDown:
|
|
word := dvInProgress[st.Phase]
|
|
if st.Progress != "" && st.Progress != "N/A" {
|
|
word += ", " + st.Progress
|
|
}
|
|
r.State, r.Word = report.StateStarting, word
|
|
}
|
|
if len(r.Facts) == 0 {
|
|
r.Facts = nil
|
|
}
|
|
}
|
|
|
|
// dvWorker is a pod CDI runs to fill a DataVolume's volume (importer,
|
|
// cloner, uploader): the volume's record says how that goes.
|
|
func dvWorker(p *pod, dvs map[string]*dataVolume) bool {
|
|
o := p.Metadata.controllerRef()
|
|
return o != nil && o.Kind == "PersistentVolumeClaim" && dvs[p.Metadata.Namespace+"/"+o.Name] != nil
|
|
}
|
|
|
|
// launcherPod is a virt-launcher pod: its VM stands for it.
|
|
func launcherPod(p *pod) bool {
|
|
o := p.Metadata.controllerRef()
|
|
return o != nil && o.Kind == "VirtualMachineInstance"
|
|
}
|