release / release (push) Successful in 7m6s
The new Cairn OBS: a self-hosted monitor for networks, hosts and services. Licensed AGPLv3.
167 lines
6.0 KiB
Go
167 lines
6.0 KiB
Go
package clusterwatch
|
|
|
|
// OpenShift and OKD (docs/cluster-watcher.md, plan step 3): the cluster's
|
|
// version and whether an update is under way, its operators, and its
|
|
// machine config pools, sent as parts of the cluster (kind cluster, named
|
|
// "clusterversion/version", "clusteroperator/ingress",
|
|
// "machineconfigpool/worker"). A node the machine config operator is
|
|
// updating is Updating, not down, even before it's cordoned.
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/cairnobs/cairnobs/internal/report"
|
|
)
|
|
|
|
// The machine config operator's annotations on a node.
|
|
const (
|
|
mcoState = "machineconfiguration.openshift.io/state" // Done, Working or Degraded
|
|
mcoReason = "machineconfiguration.openshift.io/reason"
|
|
mcoCurrent = "machineconfiguration.openshift.io/currentConfig"
|
|
mcoDesired = "machineconfiguration.openshift.io/desiredConfig"
|
|
)
|
|
|
|
// part starts a record for one of the cluster's parts.
|
|
func (b *builder) part(name string, m meta) report.Workload {
|
|
r := b.rec(report.KindCluster, name, m)
|
|
r.Namespace, r.Labels = "", nil
|
|
return r
|
|
}
|
|
|
|
// condWord is a condition's reason and message, short: "Degraded:
|
|
// IngressDegraded: The ingress controller…".
|
|
func condWord(prefix string, c *condition) string {
|
|
w := prefix
|
|
if c.Reason != "" {
|
|
w += ": " + c.Reason
|
|
}
|
|
if c.Message != "" {
|
|
w += ": " + c.Message
|
|
}
|
|
return short(w, 200)
|
|
}
|
|
|
|
// distribution names OpenShift's community build by its version string.
|
|
func distribution(version string) string {
|
|
if strings.Contains(version, "okd") {
|
|
return "OKD"
|
|
}
|
|
return "OpenShift"
|
|
}
|
|
|
|
func (b *builder) clusterVersion(cv *clusterVersion) report.Workload {
|
|
r := b.part("clusterversion/"+cv.Metadata.Name, cv.Metadata)
|
|
current := ""
|
|
for _, h := range cv.Status.History { // newest first
|
|
if h.State == "Completed" {
|
|
current = h.Version
|
|
break
|
|
}
|
|
}
|
|
desired := cv.Status.Desired.Version
|
|
if current == "" {
|
|
current = desired
|
|
}
|
|
r.Facts = map[string]any{"version": current, "distribution": distribution(current + desired)}
|
|
if cv.Spec.Channel != "" {
|
|
r.Facts["channel"] = cv.Spec.Channel
|
|
}
|
|
progressing := findCond(cv.Status.Conditions, "Progressing")
|
|
failing := findCond(cv.Status.Conditions, "Failing")
|
|
updating := progressing != nil && progressing.Status == "True"
|
|
if updating && desired != "" && desired != current {
|
|
r.Facts["desired"] = desired
|
|
}
|
|
switch {
|
|
case failing != nil && failing.Status == "True" && updating:
|
|
r.State, r.Since = report.StateDegraded, failing.LastTransitionTime
|
|
r.Word = short(fmt.Sprintf("Update to %s failing: %s", desired, failing.Message), 200)
|
|
case failing != nil && failing.Status == "True":
|
|
r.State, r.Word, r.Since = report.StateDegraded, condWord("Failing", failing), failing.LastTransitionTime
|
|
case updating:
|
|
r.State, r.Since = report.StateUpdating, progressing.LastTransitionTime
|
|
r.Word = "Updating"
|
|
if desired != "" && desired != current {
|
|
r.Word += " to " + desired
|
|
}
|
|
if progressing.Message != "" {
|
|
r.Word = short(r.Word+": "+progressing.Message, 200)
|
|
}
|
|
default:
|
|
r.State, r.Word = report.StateOK, distribution(current)+" "+current
|
|
}
|
|
return r
|
|
}
|
|
|
|
func (b *builder) clusterOperator(co *clusterOperator) report.Workload {
|
|
r := b.part("clusteroperator/"+co.Metadata.Name, co.Metadata)
|
|
for _, v := range co.Status.Versions {
|
|
if v.Name == "operator" {
|
|
r.Facts = map[string]any{"version": v.Version}
|
|
}
|
|
}
|
|
available := findCond(co.Status.Conditions, "Available")
|
|
degraded := findCond(co.Status.Conditions, "Degraded")
|
|
progressing := findCond(co.Status.Conditions, "Progressing")
|
|
switch {
|
|
case available == nil:
|
|
r.State, r.Word = report.StateStarting, "No Available condition yet"
|
|
case available.Status != "True":
|
|
r.State, r.Word, r.Since = report.StateDown, condWord("Not available", available), available.LastTransitionTime
|
|
case degraded != nil && degraded.Status == "True":
|
|
r.State, r.Word, r.Since = report.StateDegraded, condWord("Degraded", degraded), degraded.LastTransitionTime
|
|
case progressing != nil && progressing.Status == "True":
|
|
r.State, r.Word, r.Since = report.StateUpdating, condWord("Progressing", progressing), progressing.LastTransitionTime
|
|
default:
|
|
r.State, r.Word, r.Since = report.StateOK, "Available", available.LastTransitionTime
|
|
}
|
|
return r
|
|
}
|
|
|
|
func (b *builder) machineConfigPool(p *machineConfigPool) report.Workload {
|
|
r := b.part("machineconfigpool/"+p.Metadata.Name, p.Metadata)
|
|
st := p.Status
|
|
r.Facts = map[string]any{"nodes": st.MachineCount, "updated": st.UpdatedMachineCount, "ready": st.ReadyMachineCount}
|
|
if st.DegradedMachineCount > 0 {
|
|
r.Facts["degraded"] = st.DegradedMachineCount
|
|
}
|
|
degraded := findCond(st.Conditions, "Degraded")
|
|
updating := findCond(st.Conditions, "Updating")
|
|
updated := findCond(st.Conditions, "Updated")
|
|
switch {
|
|
case degraded != nil && degraded.Status == "True":
|
|
r.State, r.Word, r.Since = report.StateDegraded, condWord("Degraded", degraded), degraded.LastTransitionTime
|
|
case p.Spec.Paused && (updated == nil || updated.Status != "True"):
|
|
r.State, r.Word = report.StateOK, fmt.Sprintf("Paused, with an update waiting (%d of %d nodes updated)", st.UpdatedMachineCount, st.MachineCount)
|
|
case p.Spec.Paused:
|
|
r.State, r.Word = report.StateOK, "Paused"
|
|
case updating != nil && updating.Status == "True":
|
|
r.State, r.Since = report.StateUpdating, updating.LastTransitionTime
|
|
r.Word = fmt.Sprintf("Updating: %d of %d nodes done", st.UpdatedMachineCount, st.MachineCount)
|
|
default:
|
|
r.State, r.Word = report.StateOK, fmt.Sprintf("Up to date (%s)", nodesWord(st.MachineCount))
|
|
}
|
|
return r
|
|
}
|
|
|
|
func nodesWord(n int) string {
|
|
if n == 1 {
|
|
return "1 node"
|
|
}
|
|
return fmt.Sprintf("%d nodes", n)
|
|
}
|
|
|
|
// mcoUpdating says whether the machine config operator is working on a
|
|
// node, and its word for it; degraded says the operator couldn't.
|
|
func mcoUpdating(m meta) (updating, degraded bool, word string) {
|
|
a := m.Annotations
|
|
switch {
|
|
case a[mcoState] == "Degraded":
|
|
return false, true, short("Machine config degraded: "+a[mcoReason], 200)
|
|
case a[mcoState] == "Working", a[mcoCurrent] != "" && a[mcoDesired] != "" && a[mcoCurrent] != a[mcoDesired]:
|
|
return true, false, "Updating its machine config"
|
|
}
|
|
return false, false, ""
|
|
}
|