Files
cairnobs/internal/backup/backup.go
T
jcoffey-dev f87b405a1a agent: builds for Windows, reports the host, runs as a service
The root helper, systemd paths and statfs go behind build tags. On Windows
the collector reads CPU, memory, fixed volumes, hardware interfaces' bytes,
the OS and build and the SMBIOS UUID from kernel32 and iphlpapi (no load
average there). `obs-beacon-agent service install|uninstall` copies it
under Program Files, registers the service as its own virtual account,
restarts it on failure, logs to the Application event log, and gives
C:\ProgramData\obs-beacon to SYSTEM, Administrators and that account only.
make test vets the Windows build too.
2026-11-14 21:42:10 -07:00

308 lines
8.7 KiB
Go

// Package backup is Cairn OBS's backup check: backups.yaml says what each host
// should receive from the others, what it should make and which jobs make
// it; Scan looks (as root, on a timer, with no network) and obs-beacond judges.
// It took over from hosts-baseline's backup-check.py.
package backup
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
"gopkg.in/yaml.v3"
"github.com/cairnobs/cairnobs/internal/report"
)
// File is backups.yaml. The same file goes to every host; each host's scan
// reads its own section, and obs-beacond judges every section.
type File struct {
MaxAge time.Duration `yaml:"max_age"` // newest copy older than this; default 26h
Shrink float64 `yaml:"shrink"` // newest under this share of the one before; default 0.5
DiskWarn float64 `yaml:"disk_warn"` // a backup disk fuller than this share; default 0.9
Hosts map[string]Host `yaml:"hosts"`
}
// Host is what one host receives, makes and runs.
type Host struct {
Received []Source `yaml:"received" json:"received"`
Made []Made `yaml:"made" json:"made"`
Jobs []string `yaml:"jobs" json:"jobs"` // systemd services, e.g. gitea-backup.service
}
// Source is one sending host's directory: a subdirectory per backup, each
// holding dated copies.
type Source struct {
From string `yaml:"from" json:"from"`
Root string `yaml:"root" json:"root"`
Expect []string `yaml:"expect" json:"expect"`
MaxAge time.Duration `yaml:"max_age" json:"max_age_ns"` // default: the file's
}
// Made is a backup whichever host is in charge makes (the database dump
// comes from the PostgreSQL leader): the newest file over every pattern.
type Made struct {
Name string `yaml:"name" json:"name"`
Paths []string `yaml:"paths" json:"paths"` // glob patterns
MaxAge time.Duration `yaml:"max_age" json:"max_age_ns"`
}
// Load reads and checks backups.yaml.
func Load(p string) (*File, error) {
b, err := os.ReadFile(p)
if err != nil {
return nil, err
}
var f File
dec := yaml.NewDecoder(bytes.NewReader(b))
dec.KnownFields(true)
if err := dec.Decode(&f); err != nil {
return nil, fmt.Errorf("%s: %w", p, err)
}
if f.MaxAge == 0 {
f.MaxAge = 26 * time.Hour
}
if f.Shrink == 0 {
f.Shrink = 0.5
}
if f.DiskWarn == 0 {
f.DiskWarn = 0.9
}
for name, h := range f.Hosts {
for i := range h.Received {
s := &h.Received[i]
if s.From == "" || !filepath.IsAbs(s.Root) || len(s.Expect) == 0 {
return nil, fmt.Errorf("%s: %s: each received entry needs from, an absolute root and expect", p, name)
}
if s.MaxAge == 0 {
s.MaxAge = f.MaxAge
}
}
for i := range h.Made {
m := &h.Made[i]
if m.Name == "" || len(m.Paths) == 0 {
return nil, fmt.Errorf("%s: %s: each made entry needs a name and paths", p, name)
}
for _, g := range m.Paths {
if _, err := filepath.Match(g, ""); err != nil || !filepath.IsAbs(g) {
return nil, fmt.Errorf("%s: %s: %s: not an absolute glob pattern", p, name, g)
}
}
if m.MaxAge == 0 {
m.MaxAge = f.MaxAge
}
}
f.Hosts[name] = h
}
return &f, nil
}
// Scan looks at one host. It only reads: directory listings and file
// sizes, never contents.
func Scan(ctx context.Context, h Host, now time.Time) report.Backups {
out := report.Backups{At: now.UTC(), Received: []report.BackupDir{}, Made: []report.BackupDir{},
Roots: []report.BackupRoot{}, Jobs: []report.BackupJob{}, RAID: []report.RAIDArray{}, SMART: []report.DiskHealth{}}
fail := func(err error) { out.Errors = append(out.Errors, err.Error()) }
scanFiles(h, &out)
for _, u := range h.Jobs {
j, err := job(ctx, u)
if err != nil {
fail(err)
}
out.Jobs = append(out.Jobs, j)
}
raid, err := mdstat("/proc/mdstat")
if err != nil && !errors.Is(err, os.ErrNotExist) {
fail(err)
}
out.RAID = append(out.RAID, raid...)
smart, err := smartAll(ctx)
if err != nil {
fail(err)
}
out.SMART = append(out.SMART, smart...)
return out
}
// scanFiles fills in the received and made backups and their disks.
func scanFiles(h Host, out *report.Backups) {
fail := func(err error) { out.Errors = append(out.Errors, err.Error()) }
for _, s := range h.Received {
for _, name := range s.Expect {
d := newest(filepath.Join(s.Root, name))
d.Name, d.From = name, s.From
out.Received = append(out.Received, d)
}
if pct, err := usedPct(s.Root); err != nil {
fail(fmt.Errorf("%s: %w", s.Root, err))
} else if pct >= 0 {
out.Roots = append(out.Roots, report.BackupRoot{Path: s.Root, UsedPct: pct})
}
}
for _, m := range h.Made {
var files []string
for _, g := range m.Paths {
got, _ := filepath.Glob(g)
files = append(files, got...)
}
d := newestOf(files)
d.Name = m.Name
out.Made = append(out.Made, d)
}
}
// newest is the newest regular file in dir; hidden files are skipped
// (rsync's temporaries start with a dot).
func newest(dir string) report.BackupDir {
ents, err := os.ReadDir(dir)
if err != nil {
return report.BackupDir{Path: dir}
}
var files []string
for _, e := range ents {
if e.Type().IsRegular() && !strings.HasPrefix(e.Name(), ".") {
files = append(files, filepath.Join(dir, e.Name()))
}
}
d := newestOf(files)
if d.Count == 0 {
d.Path = dir
}
return d
}
func newestOf(files []string) report.BackupDir {
type f struct {
path string
size int64
mod time.Time
}
var list []f
for _, p := range files {
st, err := os.Stat(p)
if err != nil || !st.Mode().IsRegular() {
continue
}
list = append(list, f{p, st.Size(), st.ModTime()})
}
if len(list) == 0 {
return report.BackupDir{}
}
sort.Slice(list, func(i, j int) bool { return list[i].mod.After(list[j].mod) })
d := report.BackupDir{Path: list[0].path, Size: list[0].size, Modified: list[0].mod.UTC(), Count: len(list)}
if len(list) > 1 {
d.PrevSize = list[1].size
}
return d
}
func job(ctx context.Context, unit string) (report.BackupJob, error) {
j := report.BackupJob{Unit: unit}
out, err := exec.CommandContext(ctx, "systemctl", "show", unit, "--timestamp=unix",
"-p", "Result", "-p", "ActiveState", "-p", "LoadState", "-p", "InactiveEnterTimestamp").Output()
if err != nil {
return j, fmt.Errorf("systemctl show %s: %w", unit, err)
}
p := map[string]string{}
for _, l := range strings.Split(string(out), "\n") {
if k, v, ok := strings.Cut(l, "="); ok {
p[k] = v
}
}
if p["LoadState"] == "not-found" {
return j, nil // Result stays empty: obs-beacond says the job is missing
}
j.Result, j.Active = p["Result"], p["ActiveState"]
if s, ok := strings.CutPrefix(p["InactiveEnterTimestamp"], "@"); ok {
if n, err := strconv.ParseInt(s, 10, 64); err == nil && n > 0 {
j.Finished = time.Unix(n, 0).UTC()
}
}
return j, nil
}
// mdstat reads each array's member state: "[UU]" is healthy, an
// underscore is a missing member.
func mdstat(p string) ([]report.RAIDArray, error) {
f, err := os.Open(p)
if err != nil {
return nil, err
}
defer f.Close()
var out []report.RAIDArray
dev := ""
sc := bufio.NewScanner(f)
for sc.Scan() {
l := sc.Text()
if strings.HasPrefix(l, "md") {
dev, _, _ = strings.Cut(l, " ")
continue
}
if dev != "" && strings.Contains(l, "blocks") {
if i := strings.LastIndex(l, "["); i >= 0 && strings.HasSuffix(strings.TrimSpace(l), "]") {
out = append(out, report.RAIDArray{Dev: dev, State: strings.TrimSpace(l[i:])})
dev = ""
}
}
}
return out, sc.Err()
}
func smartAll(ctx context.Context) ([]report.DiskHealth, error) {
b, err := exec.CommandContext(ctx, "lsblk", "-dnJ", "-o", "NAME,TYPE").Output()
if err != nil {
return nil, fmt.Errorf("lsblk: %w", err)
}
var l struct {
Devices []struct{ Name, Type string } `json:"blockdevices"`
}
if err := json.Unmarshal(b, &l); err != nil {
return nil, fmt.Errorf("lsblk: %w", err)
}
var out []report.DiskHealth
for _, d := range l.Devices {
if d.Type == "disk" {
out = append(out, smart(ctx, d.Name))
}
}
return out, nil
}
// smart asks smartctl for a disk's overall health. Its exit status is a
// bit mask that's often non-zero on healthy disks, so the JSON decides.
func smart(ctx context.Context, disk string) report.DiskHealth {
h := report.DiskHealth{Disk: disk}
b, err := exec.CommandContext(ctx, "smartctl", "-H", "-j", "/dev/"+disk).Output()
var r struct {
Status *struct {
Passed bool `json:"passed"`
} `json:"smart_status"`
Smartctl struct {
Messages []struct {
String string `json:"string"`
} `json:"messages"`
} `json:"smartctl"`
}
if jerr := json.Unmarshal(b, &r); jerr != nil || r.Status == nil {
h.Error = "smartctl gave no verdict"
if len(r.Smartctl.Messages) > 0 {
h.Error = r.Smartctl.Messages[0].String
} else if err != nil {
h.Error = err.Error()
}
return h
}
h.OK = r.Status.Passed
return h
}