The root helper, systemd paths and statfs go behind build tags. On Windows the collector reads CPU, memory, fixed volumes, hardware interfaces' bytes, the OS and build and the SMBIOS UUID from kernel32 and iphlpapi (no load average there). `obs-beacon-agent service install|uninstall` copies it under Program Files, registers the service as its own virtual account, restarts it on failure, logs to the Application event log, and gives C:\ProgramData\obs-beacon to SYSTEM, Administrators and that account only. make test vets the Windows build too.
308 lines
8.7 KiB
Go
308 lines
8.7 KiB
Go
// Package backup is Cairn OBS's backup check: backups.yaml says what each host
|
|
// should receive from the others, what it should make and which jobs make
|
|
// it; Scan looks (as root, on a timer, with no network) and obs-beacond judges.
|
|
// It took over from hosts-baseline's backup-check.py.
|
|
package backup
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"gopkg.in/yaml.v3"
|
|
|
|
"github.com/cairnobs/cairnobs/internal/report"
|
|
)
|
|
|
|
// File is backups.yaml. The same file goes to every host; each host's scan
|
|
// reads its own section, and obs-beacond judges every section.
|
|
type File struct {
|
|
MaxAge time.Duration `yaml:"max_age"` // newest copy older than this; default 26h
|
|
Shrink float64 `yaml:"shrink"` // newest under this share of the one before; default 0.5
|
|
DiskWarn float64 `yaml:"disk_warn"` // a backup disk fuller than this share; default 0.9
|
|
Hosts map[string]Host `yaml:"hosts"`
|
|
}
|
|
|
|
// Host is what one host receives, makes and runs.
|
|
type Host struct {
|
|
Received []Source `yaml:"received" json:"received"`
|
|
Made []Made `yaml:"made" json:"made"`
|
|
Jobs []string `yaml:"jobs" json:"jobs"` // systemd services, e.g. gitea-backup.service
|
|
}
|
|
|
|
// Source is one sending host's directory: a subdirectory per backup, each
|
|
// holding dated copies.
|
|
type Source struct {
|
|
From string `yaml:"from" json:"from"`
|
|
Root string `yaml:"root" json:"root"`
|
|
Expect []string `yaml:"expect" json:"expect"`
|
|
MaxAge time.Duration `yaml:"max_age" json:"max_age_ns"` // default: the file's
|
|
}
|
|
|
|
// Made is a backup whichever host is in charge makes (the database dump
|
|
// comes from the PostgreSQL leader): the newest file over every pattern.
|
|
type Made struct {
|
|
Name string `yaml:"name" json:"name"`
|
|
Paths []string `yaml:"paths" json:"paths"` // glob patterns
|
|
MaxAge time.Duration `yaml:"max_age" json:"max_age_ns"`
|
|
}
|
|
|
|
// Load reads and checks backups.yaml.
|
|
func Load(p string) (*File, error) {
|
|
b, err := os.ReadFile(p)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var f File
|
|
dec := yaml.NewDecoder(bytes.NewReader(b))
|
|
dec.KnownFields(true)
|
|
if err := dec.Decode(&f); err != nil {
|
|
return nil, fmt.Errorf("%s: %w", p, err)
|
|
}
|
|
if f.MaxAge == 0 {
|
|
f.MaxAge = 26 * time.Hour
|
|
}
|
|
if f.Shrink == 0 {
|
|
f.Shrink = 0.5
|
|
}
|
|
if f.DiskWarn == 0 {
|
|
f.DiskWarn = 0.9
|
|
}
|
|
for name, h := range f.Hosts {
|
|
for i := range h.Received {
|
|
s := &h.Received[i]
|
|
if s.From == "" || !filepath.IsAbs(s.Root) || len(s.Expect) == 0 {
|
|
return nil, fmt.Errorf("%s: %s: each received entry needs from, an absolute root and expect", p, name)
|
|
}
|
|
if s.MaxAge == 0 {
|
|
s.MaxAge = f.MaxAge
|
|
}
|
|
}
|
|
for i := range h.Made {
|
|
m := &h.Made[i]
|
|
if m.Name == "" || len(m.Paths) == 0 {
|
|
return nil, fmt.Errorf("%s: %s: each made entry needs a name and paths", p, name)
|
|
}
|
|
for _, g := range m.Paths {
|
|
if _, err := filepath.Match(g, ""); err != nil || !filepath.IsAbs(g) {
|
|
return nil, fmt.Errorf("%s: %s: %s: not an absolute glob pattern", p, name, g)
|
|
}
|
|
}
|
|
if m.MaxAge == 0 {
|
|
m.MaxAge = f.MaxAge
|
|
}
|
|
}
|
|
f.Hosts[name] = h
|
|
}
|
|
return &f, nil
|
|
}
|
|
|
|
// Scan looks at one host. It only reads: directory listings and file
|
|
// sizes, never contents.
|
|
func Scan(ctx context.Context, h Host, now time.Time) report.Backups {
|
|
out := report.Backups{At: now.UTC(), Received: []report.BackupDir{}, Made: []report.BackupDir{},
|
|
Roots: []report.BackupRoot{}, Jobs: []report.BackupJob{}, RAID: []report.RAIDArray{}, SMART: []report.DiskHealth{}}
|
|
fail := func(err error) { out.Errors = append(out.Errors, err.Error()) }
|
|
scanFiles(h, &out)
|
|
for _, u := range h.Jobs {
|
|
j, err := job(ctx, u)
|
|
if err != nil {
|
|
fail(err)
|
|
}
|
|
out.Jobs = append(out.Jobs, j)
|
|
}
|
|
raid, err := mdstat("/proc/mdstat")
|
|
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
fail(err)
|
|
}
|
|
out.RAID = append(out.RAID, raid...)
|
|
smart, err := smartAll(ctx)
|
|
if err != nil {
|
|
fail(err)
|
|
}
|
|
out.SMART = append(out.SMART, smart...)
|
|
return out
|
|
}
|
|
|
|
// scanFiles fills in the received and made backups and their disks.
|
|
func scanFiles(h Host, out *report.Backups) {
|
|
fail := func(err error) { out.Errors = append(out.Errors, err.Error()) }
|
|
for _, s := range h.Received {
|
|
for _, name := range s.Expect {
|
|
d := newest(filepath.Join(s.Root, name))
|
|
d.Name, d.From = name, s.From
|
|
out.Received = append(out.Received, d)
|
|
}
|
|
if pct, err := usedPct(s.Root); err != nil {
|
|
fail(fmt.Errorf("%s: %w", s.Root, err))
|
|
} else if pct >= 0 {
|
|
out.Roots = append(out.Roots, report.BackupRoot{Path: s.Root, UsedPct: pct})
|
|
}
|
|
}
|
|
for _, m := range h.Made {
|
|
var files []string
|
|
for _, g := range m.Paths {
|
|
got, _ := filepath.Glob(g)
|
|
files = append(files, got...)
|
|
}
|
|
d := newestOf(files)
|
|
d.Name = m.Name
|
|
out.Made = append(out.Made, d)
|
|
}
|
|
}
|
|
|
|
// newest is the newest regular file in dir; hidden files are skipped
|
|
// (rsync's temporaries start with a dot).
|
|
func newest(dir string) report.BackupDir {
|
|
ents, err := os.ReadDir(dir)
|
|
if err != nil {
|
|
return report.BackupDir{Path: dir}
|
|
}
|
|
var files []string
|
|
for _, e := range ents {
|
|
if e.Type().IsRegular() && !strings.HasPrefix(e.Name(), ".") {
|
|
files = append(files, filepath.Join(dir, e.Name()))
|
|
}
|
|
}
|
|
d := newestOf(files)
|
|
if d.Count == 0 {
|
|
d.Path = dir
|
|
}
|
|
return d
|
|
}
|
|
|
|
func newestOf(files []string) report.BackupDir {
|
|
type f struct {
|
|
path string
|
|
size int64
|
|
mod time.Time
|
|
}
|
|
var list []f
|
|
for _, p := range files {
|
|
st, err := os.Stat(p)
|
|
if err != nil || !st.Mode().IsRegular() {
|
|
continue
|
|
}
|
|
list = append(list, f{p, st.Size(), st.ModTime()})
|
|
}
|
|
if len(list) == 0 {
|
|
return report.BackupDir{}
|
|
}
|
|
sort.Slice(list, func(i, j int) bool { return list[i].mod.After(list[j].mod) })
|
|
d := report.BackupDir{Path: list[0].path, Size: list[0].size, Modified: list[0].mod.UTC(), Count: len(list)}
|
|
if len(list) > 1 {
|
|
d.PrevSize = list[1].size
|
|
}
|
|
return d
|
|
}
|
|
|
|
func job(ctx context.Context, unit string) (report.BackupJob, error) {
|
|
j := report.BackupJob{Unit: unit}
|
|
out, err := exec.CommandContext(ctx, "systemctl", "show", unit, "--timestamp=unix",
|
|
"-p", "Result", "-p", "ActiveState", "-p", "LoadState", "-p", "InactiveEnterTimestamp").Output()
|
|
if err != nil {
|
|
return j, fmt.Errorf("systemctl show %s: %w", unit, err)
|
|
}
|
|
p := map[string]string{}
|
|
for _, l := range strings.Split(string(out), "\n") {
|
|
if k, v, ok := strings.Cut(l, "="); ok {
|
|
p[k] = v
|
|
}
|
|
}
|
|
if p["LoadState"] == "not-found" {
|
|
return j, nil // Result stays empty: obs-beacond says the job is missing
|
|
}
|
|
j.Result, j.Active = p["Result"], p["ActiveState"]
|
|
if s, ok := strings.CutPrefix(p["InactiveEnterTimestamp"], "@"); ok {
|
|
if n, err := strconv.ParseInt(s, 10, 64); err == nil && n > 0 {
|
|
j.Finished = time.Unix(n, 0).UTC()
|
|
}
|
|
}
|
|
return j, nil
|
|
}
|
|
|
|
// mdstat reads each array's member state: "[UU]" is healthy, an
|
|
// underscore is a missing member.
|
|
func mdstat(p string) ([]report.RAIDArray, error) {
|
|
f, err := os.Open(p)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer f.Close()
|
|
var out []report.RAIDArray
|
|
dev := ""
|
|
sc := bufio.NewScanner(f)
|
|
for sc.Scan() {
|
|
l := sc.Text()
|
|
if strings.HasPrefix(l, "md") {
|
|
dev, _, _ = strings.Cut(l, " ")
|
|
continue
|
|
}
|
|
if dev != "" && strings.Contains(l, "blocks") {
|
|
if i := strings.LastIndex(l, "["); i >= 0 && strings.HasSuffix(strings.TrimSpace(l), "]") {
|
|
out = append(out, report.RAIDArray{Dev: dev, State: strings.TrimSpace(l[i:])})
|
|
dev = ""
|
|
}
|
|
}
|
|
}
|
|
return out, sc.Err()
|
|
}
|
|
|
|
func smartAll(ctx context.Context) ([]report.DiskHealth, error) {
|
|
b, err := exec.CommandContext(ctx, "lsblk", "-dnJ", "-o", "NAME,TYPE").Output()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("lsblk: %w", err)
|
|
}
|
|
var l struct {
|
|
Devices []struct{ Name, Type string } `json:"blockdevices"`
|
|
}
|
|
if err := json.Unmarshal(b, &l); err != nil {
|
|
return nil, fmt.Errorf("lsblk: %w", err)
|
|
}
|
|
var out []report.DiskHealth
|
|
for _, d := range l.Devices {
|
|
if d.Type == "disk" {
|
|
out = append(out, smart(ctx, d.Name))
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// smart asks smartctl for a disk's overall health. Its exit status is a
|
|
// bit mask that's often non-zero on healthy disks, so the JSON decides.
|
|
func smart(ctx context.Context, disk string) report.DiskHealth {
|
|
h := report.DiskHealth{Disk: disk}
|
|
b, err := exec.CommandContext(ctx, "smartctl", "-H", "-j", "/dev/"+disk).Output()
|
|
var r struct {
|
|
Status *struct {
|
|
Passed bool `json:"passed"`
|
|
} `json:"smart_status"`
|
|
Smartctl struct {
|
|
Messages []struct {
|
|
String string `json:"string"`
|
|
} `json:"messages"`
|
|
} `json:"smartctl"`
|
|
}
|
|
if jerr := json.Unmarshal(b, &r); jerr != nil || r.Status == nil {
|
|
h.Error = "smartctl gave no verdict"
|
|
if len(r.Smartctl.Messages) > 0 {
|
|
h.Error = r.Smartctl.Messages[0].String
|
|
} else if err != nil {
|
|
h.Error = err.Error()
|
|
}
|
|
return h
|
|
}
|
|
h.OK = r.Status.Passed
|
|
return h
|
|
}
|