Files
cairnobs/internal/server/cloudflare.go
T
jcoffey-dev db156c4f06
release / release (push) Successful in 7m6s
Cairn OBS
The new Cairn OBS: a self-hosted monitor for networks, hosts and
services. Licensed AGPLv3.
2026-11-14 03:27:09 -07:00

280 lines
9.3 KiB
Go

package server
import (
"encoding/json"
"fmt"
"log"
"net/http"
"sort"
"strings"
"sync"
"time"
"github.com/cairnobs/cairnobs/internal/cloudflare"
"github.com/cairnobs/cairnobs/internal/state"
)
// cloudflareConditions judges the latest Cloudflare reading. The status
// page is shown but never alerts: a Cloudflare incident somewhere in the
// world usually isn't ours, and when it is, the pools say so.
func (srv *Server) cloudflareConditions(now time.Time) []Condition {
if srv.cf == nil {
return nil
}
s, cfg := srv.cf.Snapshot(), srv.cf.Config()
const href = "/cloudflare"
var out []Condition
if s.Error != "" && now.Sub(s.ErrorSince) >= 3*cfg.Every {
out = append(out, Condition{Key: "cloudflare/api", Severity: "warning", Title: "Can't read Cloudflare",
Detail: s.Error + ". Load balancer, SSL and certificate checks are on hold.", Href: href})
}
if s.Checked.IsZero() {
return out
}
for _, p := range s.Pools {
if !p.Enabled {
continue
}
if !p.Healthy {
out = append(out, Condition{Key: "cloudflare/pool/" + p.Name, Severity: "critical",
Title: "Cloudflare pool " + p.Name + " has no healthy origin",
Detail: "Every origin is failing Cloudflare's health check, so the sites behind it are down or on a fallback.", Href: href})
}
for _, o := range p.Origins {
if !o.Enabled || o.Healthy {
continue
}
detail := o.Reason
if o.Code != 0 {
detail += fmt.Sprintf(", HTTP %d", o.Code)
}
out = append(out, Condition{Key: "cloudflare/origin/" + p.Name + "/" + o.Name, Severity: "warning",
Title: fmt.Sprintf("%s is failing %s's health check", o.Name, p.Name),
Detail: detail + ". Cloudflare sends its share of traffic to the other origins.", Host: o.Name, Href: href})
}
}
for _, z := range s.Zones {
if t := z.Traffic; t != nil && t.Requests > 0 {
pct := 100 * float64(t.Errors) / float64(t.Requests)
sev := ""
switch {
case t.Errors >= cfg.ErrorMin && pct >= cfg.ErrorCriticalPct:
sev = "critical"
case t.Errors >= cfg.ErrorMin && pct >= cfg.ErrorWarnPct:
sev = "warning"
}
if sev != "" {
detail := fmt.Sprintf("%d of %d requests in the last 15 minutes were answered with a server error", t.Errors, t.Requests)
if len(t.ErrorHosts) > 0 {
detail += fmt.Sprintf(", most on %s (%d)", t.ErrorHosts[0].Host, t.ErrorHosts[0].Count)
}
out = append(out, Condition{Key: "cloudflare/errors/" + z.Name, Severity: sev,
Title: fmt.Sprintf("%s: %.0f%% of requests failing", z.Name, pct), Detail: detail + ".", Href: href})
}
usual := float64(t.BlockedDay) / 96 // the day before, per 15 minutes
if t.Blocked >= cfg.BlockMin && float64(t.Blocked) >= cfg.BlockFactor*usual {
out = append(out, Condition{Key: "cloudflare/blocks/" + z.Name, Severity: "warning",
Title: fmt.Sprintf("Cloudflare is blocking a lot of %s traffic", z.Name),
Detail: fmt.Sprintf("%d requests blocked or challenged in the last 15 minutes; usually about %.0f. An attack, a bot, or a rule catching real visitors.", t.Blocked, usual), Href: href})
}
}
if added, removed := srv.dns.diff(z.Name, records(z.Records)); len(added)+len(removed) > 0 {
var lines []string
for _, r := range removed {
lines = append(lines, "removed "+r)
}
for _, r := range added {
lines = append(lines, "added "+r)
}
if len(lines) > 4 {
lines = append(lines[:3], fmt.Sprintf("and %d more", len(lines)-3))
}
out = append(out, Condition{Key: "cloudflare/dns/" + z.Name, Severity: "warning",
Title: fmt.Sprintf("DNS for %s changed (%d added, %d removed)", z.Name, len(added), len(removed)),
Detail: strings.Join(lines, "; ") + ". Accept it on the Cloudflare page if it was meant.", Href: href})
}
if want := cfg.WantSSL(z.Name); z.SSL != "" && z.SSL != want {
out = append(out, Condition{Key: "cloudflare/ssl/" + z.Name, Severity: "warning",
Title: fmt.Sprintf("%s SSL mode is %s, not %s", z.Name, z.SSL, want),
Detail: "Cloudflare's SSL/TLS encryption mode for the zone changed.", Href: href})
}
for _, c := range z.Certs {
key := "cloudflare/cert/" + z.Name + "/" + strings.Join(c.Hosts, ",")
days := int(c.Expires.Sub(now).Hours() / 24)
switch {
case c.Status != "active":
out = append(out, Condition{Key: key, Severity: "warning",
Title: fmt.Sprintf("%s edge certificate is %s", z.Name, strings.ReplaceAll(c.Status, "_", " ")),
Detail: "Covers " + strings.Join(c.Hosts, ", ") + ".", Href: href})
case c.Expires.IsZero():
case days < cfg.CertCriticalDays:
out = append(out, Condition{Key: key, Severity: "critical",
Title: fmt.Sprintf("%s edge certificate expires in %s", z.Name, plural(days, "day", "days")),
Detail: "Cloudflare normally renews well before this; renewal is stuck. Covers " + strings.Join(c.Hosts, ", ") + ".", Href: href})
case days < cfg.CertWarnDays:
out = append(out, Condition{Key: key, Severity: "warning",
Title: fmt.Sprintf("%s edge certificate expires in %s", z.Name, plural(days, "day", "days")),
Detail: "Cloudflare normally renews 30 days ahead. Covers " + strings.Join(c.Hosts, ", ") + ".", Href: href})
}
}
}
return out
}
func (srv *Server) handleCloudflare(w http.ResponseWriter, r *http.Request) {
if srv.cf == nil {
writeJSON(w, http.StatusOK, map[string]any{"configured": false})
return
}
cfg := srv.cf.Config()
snap := srv.cf.Snapshot()
changes := map[string]map[string][]string{}
for _, z := range snap.Zones {
if added, removed := srv.dns.diff(z.Name, records(z.Records)); len(added)+len(removed) > 0 {
changes[z.Name] = map[string][]string{"added": nonNilStrings(added), "removed": nonNilStrings(removed)}
}
}
writeJSON(w, http.StatusOK, map[string]any{
"configured": true, "snapshot": snap, "dns_changes": changes,
"error_warn_pct": cfg.ErrorWarnPct, "error_critical_pct": cfg.ErrorCriticalPct,
"conditions": nonNilConds(srv.cloudflareConditions(time.Now())),
"ssl_mode": cfg.SSLMode, "ssl_mode_zones": cfg.SSLModeZones, "cert_warn_days": cfg.CertWarnDays, "cert_critical_days": cfg.CertCriticalDays,
"every_s": int(cfg.Every / time.Second),
})
}
func nonNilConds(c []Condition) []Condition {
if c == nil {
return []Condition{}
}
return c
}
func (srv *Server) handleDNSAccept(w http.ResponseWriter, r *http.Request) {
if srv.cf == nil {
http.NotFound(w, r)
return
}
zone := r.PathValue("zone")
for _, z := range srv.cf.Snapshot().Zones {
if z.Name != zone {
continue
}
added, removed := srv.dns.accept(zone, records(z.Records))
who := "someone"
if _, s, ok := srv.sessions.get(r); ok {
who = s.User
}
srv.audit(who, "accepted DNS changes", fmt.Sprintf("%s: %d added, %d removed", zone, added, removed))
writeJSON(w, http.StatusOK, map[string]int{"added": added, "removed": removed})
return
}
http.NotFound(w, r)
}
// dnsBaseline holds each zone's DNS records as last accepted, by Record.Key.
// A zone seen for the first time is taken as it is. With a database the
// baseline survives restarts, so a change made while obs-beacond was down is
// still caught.
// Akamai's Edge DNS zones have one of their own (prefix "ak_dns:").
type dnsBaseline struct {
mu sync.Mutex
m map[string]map[string]string
db *state.DB
prefix string // the database key's; "cf_dns:" when empty
}
// records are Cloudflare's records as a baseline holds them: each one's
// whole setting to how it reads; nil when they were never read.
func records(recs []cloudflare.Record) map[string]string {
if recs == nil {
return nil
}
m := make(map[string]string, len(recs))
for _, r := range recs {
m[r.Key()] = r.String()
}
return m
}
func (b *dnsBaseline) key(zone string) string {
if b.prefix == "" {
return "cf_dns:" + zone
}
return b.prefix + zone
}
// base returns the zone's baseline, loading it or starting it from recs.
// Callers hold b.mu.
func (b *dnsBaseline) base(zone string, recs map[string]string) map[string]string {
if b.m == nil {
b.m = map[string]map[string]string{}
}
if m, ok := b.m[zone]; ok {
return m
}
if b.db != nil {
if v := b.db.Meta(b.key(zone)); v != "" {
var m map[string]string
if json.Unmarshal([]byte(v), &m) == nil {
b.m[zone] = m
return m
}
}
}
b.set(zone, recs)
return recs
}
func (b *dnsBaseline) set(zone string, m map[string]string) {
b.m[zone] = m
if b.db != nil {
v, _ := json.Marshal(m)
if err := b.db.SetMeta(b.key(zone), string(v)); err != nil {
log.Printf("can't store the DNS baseline for %s: %v", zone, err)
}
}
}
// diff lists what was added and removed since the baseline, readable and sorted.
func (b *dnsBaseline) diff(zone string, cur map[string]string) (added, removed []string) {
if cur == nil {
return nil, nil // never read: nothing to compare
}
b.mu.Lock()
defer b.mu.Unlock()
base := b.base(zone, cur)
for k, v := range cur {
if _, ok := base[k]; !ok {
added = append(added, v)
}
}
for k, v := range base {
if _, ok := cur[k]; !ok {
removed = append(removed, v)
}
}
sort.Strings(added)
sort.Strings(removed)
return added, removed
}
// accept makes the current records the baseline and says what that took in.
func (b *dnsBaseline) accept(zone string, cur map[string]string) (added, removed int) {
if cur == nil {
return 0, 0 // never read: nothing to accept
}
a, r := b.diff(zone, cur)
b.mu.Lock()
defer b.mu.Unlock()
b.set(zone, cur)
return len(a), len(r)
}
func nonNilStrings(s []string) []string {
if s == nil {
return []string{}
}
return s
}