SECURITY.md says where to report a problem ([email protected]) and what's in scope. The README points at docs.cairnobs.org for people installing Cairn OBS. The tests, examples and notes named a real fleet: its host names, its mesh and desk link, a public address, its site and its backups. They're now node1, node2 and so on, wg-desk, 10.30/10.31, 203.0.113.52 (TEST-NET-3), Lisbon, wiki-db and photos. docs/install.md's Windows paragraph said no VMs; the agent now reports certificate stores, the event log and Hyper-V's VMs, on amd64 and arm64.
140 lines
4.6 KiB
Go
140 lines
4.6 KiB
Go
package backup
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/cairnobs/cairnobs/internal/report"
|
|
)
|
|
|
|
// Finding is one thing wrong with a host's backups; obs-beacond makes it a
|
|
// condition. Missing or stale backups are serious (email and Discord, not
|
|
// the phone); a failing disk or degraded RAID is critical.
|
|
type Finding struct {
|
|
Key string // unique per host, e.g. received/node2/wiki-db
|
|
Severity string
|
|
Title string
|
|
Detail string
|
|
}
|
|
|
|
// Judge compares host's latest scan with what backups.yaml expects.
|
|
func (f *File) Judge(host string, b *report.Backups, now time.Time, scanEvery time.Duration) []Finding {
|
|
h, ok := f.Hosts[host]
|
|
if !ok {
|
|
return nil
|
|
}
|
|
var out []Finding
|
|
add := func(key, sev, title, detail string) {
|
|
out = append(out, Finding{Key: key, Severity: sev, Title: title, Detail: detail})
|
|
}
|
|
if b == nil {
|
|
add("scan", "warning", "No backup scan from "+host, "The obs-beacon-scan timer hasn't written a scan obs-beacon-agent could read. systemctl status obs-beacon-scan.timer")
|
|
return out
|
|
}
|
|
if age := now.Sub(b.At); age > 4*scanEvery {
|
|
add("scan", "warning", "The backup scan on "+host+" is "+ago(age)+" old",
|
|
"The obs-beacon-scan timer has stopped. systemctl status obs-beacon-scan.timer")
|
|
}
|
|
for _, e := range b.Errors {
|
|
add("scan-error/"+e, "warning", "The backup scan on "+host+" hit a problem", e+".")
|
|
}
|
|
byName := func(list []report.BackupDir, from, name string) *report.BackupDir {
|
|
for i := range list {
|
|
if list[i].Name == name && list[i].From == from {
|
|
return &list[i]
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
judge := func(key, what string, d *report.BackupDir, maxAge time.Duration) {
|
|
switch {
|
|
case d == nil || d.Count == 0:
|
|
where := ""
|
|
if d != nil {
|
|
where = " in " + d.Path
|
|
}
|
|
add(key, "serious", what+": nothing on "+host, "No copy has arrived"+where+".")
|
|
case d.Size == 0:
|
|
add(key, "serious", what+" on "+host+" is empty", d.Path+" is 0 bytes.")
|
|
case now.Sub(d.Modified) > maxAge:
|
|
add(key, "serious", fmt.Sprintf("%s on %s is %s old", what, host, ago(now.Sub(d.Modified))),
|
|
fmt.Sprintf("The newest copy is %s (%s); they should be under %s.", d.Path, Human(d.Size), ago(maxAge)))
|
|
case d.PrevSize > 0 && float64(d.Size) < f.Shrink*float64(d.PrevSize):
|
|
add(key, "warning", fmt.Sprintf("%s on %s shrank to %s", what, host, Human(d.Size)),
|
|
fmt.Sprintf("The copy before was %s. Backups grow slowly; this usually means something was left out.", Human(d.PrevSize)))
|
|
}
|
|
}
|
|
for _, s := range h.Received {
|
|
for _, name := range s.Expect {
|
|
judge("received/"+s.From+"/"+name, s.From+"'s "+name+" backup", byName(b.Received, s.From, name), s.MaxAge)
|
|
}
|
|
}
|
|
for _, m := range h.Made {
|
|
judge("made/"+m.Name, "The "+m.Name+" backup", byName(b.Made, "", m.Name), m.MaxAge)
|
|
}
|
|
for _, r := range b.Roots {
|
|
if r.UsedPct >= 100*f.DiskWarn {
|
|
add("disk/"+r.Path, "warning", fmt.Sprintf("Backup disk on %s is %.0f%% full", host, r.UsedPct),
|
|
r.Path+" is where the other hosts' backups land. Old copies or a bigger disk.")
|
|
}
|
|
}
|
|
for _, unit := range h.Jobs {
|
|
var j *report.BackupJob
|
|
for i := range b.Jobs {
|
|
if b.Jobs[i].Unit == unit {
|
|
j = &b.Jobs[i]
|
|
}
|
|
}
|
|
switch {
|
|
case j == nil:
|
|
case j.Result == "":
|
|
add("job/"+unit, "warning", unit+" is missing on "+host, "backups.yaml lists it, but systemd doesn't know it.")
|
|
case j.Result != "success":
|
|
add("job/"+unit, "serious", unit+" failed on "+host,
|
|
fmt.Sprintf("Its last run ended with %s. journalctl -u %s", strings.ReplaceAll(j.Result, "-", " "), unit))
|
|
}
|
|
}
|
|
for _, r := range b.RAID {
|
|
if strings.Contains(r.State, "_") {
|
|
add("raid/"+r.Dev, "critical", fmt.Sprintf("RAID %s on %s is degraded %s", r.Dev, host, r.State),
|
|
"A disk has dropped out of the mirror; the other one is all there is. cat /proc/mdstat")
|
|
}
|
|
}
|
|
for _, d := range b.SMART {
|
|
switch {
|
|
case d.Error != "":
|
|
add("smart/"+d.Disk, "warning", "Can't read /dev/"+d.Disk+"'s health on "+host, d.Error+".")
|
|
case !d.OK:
|
|
add("smart/"+d.Disk, "critical", "/dev/"+d.Disk+" on "+host+" is failing", "SMART's overall health check did not pass. smartctl -a /dev/"+d.Disk)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func ago(d time.Duration) string {
|
|
switch {
|
|
case d >= 48*time.Hour:
|
|
return fmt.Sprintf("%d days", int(d.Hours()/24))
|
|
case d >= 2*time.Hour:
|
|
return fmt.Sprintf("%d hours", int(d.Hours()))
|
|
default:
|
|
return fmt.Sprintf("%d minutes", int(d.Minutes()))
|
|
}
|
|
}
|
|
|
|
// Human is a size in KB, MB, GB.
|
|
func Human(n int64) string {
|
|
f := float64(n)
|
|
for _, u := range []string{"B", "KB", "MB", "GB"} {
|
|
if f < 1024 {
|
|
if u == "B" {
|
|
return fmt.Sprintf("%d B", n)
|
|
}
|
|
return fmt.Sprintf("%.1f %s", f, u)
|
|
}
|
|
f /= 1024
|
|
}
|
|
return fmt.Sprintf("%.1f TB", f)
|
|
}
|