Files
cairnobs/internal/server/cloudflare.go
T
jcoffey-dev 2abab8264d
ci / check (pull_request) Successful in 3m52s
ci / postgres (pull_request) Successful in 7m37s
Certificates: critical only in the last 48 hours
A certificate four days from its end was critical, so it repeated every
15 minutes at the same level as an outage. Now every certificate
condition carries its end, and one pass holds it at warning until it's
within certificates.critical_within (default 48h) of it, whichever source
judged it critical: hosts' ports and files, site checks, Cloudflare,
Akamai CPS, Vault and cert-manager. An ended one stays critical. The
Certificates page judges by the same window; a certificate only in
files still says nothing until 7 days out, and warns from there.

Closes #34.
2026-11-15 22:36:39 -07:00

280 lines
9.4 KiB
Go

package server
import (
"encoding/json"
"fmt"
"log"
"net/http"
"sort"
"strings"
"sync"
"time"
"github.com/cairnobs/cairnobs/internal/cloudflare"
"github.com/cairnobs/cairnobs/internal/state"
)
// cloudflareConditions judges the latest Cloudflare reading. The status
// page is shown but never alerts: a Cloudflare incident somewhere in the
// world usually isn't ours, and when it is, the pools say so.
func (srv *Server) cloudflareConditions(now time.Time) []Condition {
if srv.cf == nil {
return nil
}
s, cfg := srv.cf.Snapshot(), srv.cf.Config()
const href = "/cloudflare"
var out []Condition
if s.Error != "" && now.Sub(s.ErrorSince) >= 3*cfg.Every {
out = append(out, Condition{Key: "cloudflare/api", Severity: "warning", Title: "Can't read Cloudflare",
Detail: s.Error + ". Load balancer, SSL and certificate checks are on hold.", Href: href})
}
if s.Checked.IsZero() {
return out
}
for _, p := range s.Pools {
if !p.Enabled {
continue
}
if !p.Healthy {
out = append(out, Condition{Key: "cloudflare/pool/" + p.Name, Severity: "critical",
Title: "Cloudflare pool " + p.Name + " has no healthy origin",
Detail: "Every origin is failing Cloudflare's health check, so the sites behind it are down or on a fallback.", Href: href})
}
for _, o := range p.Origins {
if !o.Enabled || o.Healthy {
continue
}
detail := o.Reason
if o.Code != 0 {
detail += fmt.Sprintf(", HTTP %d", o.Code)
}
out = append(out, Condition{Key: "cloudflare/origin/" + p.Name + "/" + o.Name, Severity: "warning",
Title: fmt.Sprintf("%s is failing %s's health check", o.Name, p.Name),
Detail: detail + ". Cloudflare sends its share of traffic to the other origins.", Host: o.Name, Href: href})
}
}
for _, z := range s.Zones {
if t := z.Traffic; t != nil && t.Requests > 0 {
pct := 100 * float64(t.Errors) / float64(t.Requests)
sev := ""
switch {
case t.Errors >= cfg.ErrorMin && pct >= cfg.ErrorCriticalPct:
sev = "critical"
case t.Errors >= cfg.ErrorMin && pct >= cfg.ErrorWarnPct:
sev = "warning"
}
if sev != "" {
detail := fmt.Sprintf("%d of %d requests in the last 15 minutes were answered with a server error", t.Errors, t.Requests)
if len(t.ErrorHosts) > 0 {
detail += fmt.Sprintf(", most on %s (%d)", t.ErrorHosts[0].Host, t.ErrorHosts[0].Count)
}
out = append(out, Condition{Key: "cloudflare/errors/" + z.Name, Severity: sev,
Title: fmt.Sprintf("%s: %.0f%% of requests failing", z.Name, pct), Detail: detail + ".", Href: href})
}
usual := float64(t.BlockedDay) / 96 // the day before, per 15 minutes
if t.Blocked >= cfg.BlockMin && float64(t.Blocked) >= cfg.BlockFactor*usual {
out = append(out, Condition{Key: "cloudflare/blocks/" + z.Name, Severity: "warning",
Title: fmt.Sprintf("Cloudflare is blocking a lot of %s traffic", z.Name),
Detail: fmt.Sprintf("%d requests blocked or challenged in the last 15 minutes; usually about %.0f. An attack, a bot, or a rule catching real visitors.", t.Blocked, usual), Href: href})
}
}
if added, removed := srv.dns.diff(z.Name, records(z.Records)); len(added)+len(removed) > 0 {
var lines []string
for _, r := range removed {
lines = append(lines, "removed "+r)
}
for _, r := range added {
lines = append(lines, "added "+r)
}
if len(lines) > 4 {
lines = append(lines[:3], fmt.Sprintf("and %d more", len(lines)-3))
}
out = append(out, Condition{Key: "cloudflare/dns/" + z.Name, Severity: "warning",
Title: fmt.Sprintf("DNS for %s changed (%d added, %d removed)", z.Name, len(added), len(removed)),
Detail: strings.Join(lines, "; ") + ". Accept it on the Cloudflare page if it was meant.", Href: href})
}
if want := cfg.WantSSL(z.Name); z.SSL != "" && z.SSL != want {
out = append(out, Condition{Key: "cloudflare/ssl/" + z.Name, Severity: "warning",
Title: fmt.Sprintf("%s SSL mode is %s, not %s", z.Name, z.SSL, want),
Detail: "Cloudflare's SSL/TLS encryption mode for the zone changed.", Href: href})
}
for _, c := range z.Certs {
key := "cloudflare/cert/" + z.Name + "/" + strings.Join(c.Hosts, ",")
days := int(c.Expires.Sub(now).Hours() / 24)
switch {
case c.Status != "active":
out = append(out, Condition{Key: key, Severity: "warning",
Title: fmt.Sprintf("%s edge certificate is %s", z.Name, strings.ReplaceAll(c.Status, "_", " ")),
Detail: "Covers " + strings.Join(c.Hosts, ", ") + ".", Href: href})
case c.Expires.IsZero():
case days < cfg.CertCriticalDays:
out = append(out, Condition{Key: key, Severity: "critical", Ends: c.Expires,
Title: fmt.Sprintf("%s edge certificate expires in %s", z.Name, plural(days, "day", "days")),
Detail: "Cloudflare normally renews well before this; renewal is stuck. Covers " + strings.Join(c.Hosts, ", ") + ".", Href: href})
case days < cfg.CertWarnDays:
out = append(out, Condition{Key: key, Severity: "warning", Ends: c.Expires,
Title: fmt.Sprintf("%s edge certificate expires in %s", z.Name, plural(days, "day", "days")),
Detail: "Cloudflare normally renews 30 days ahead. Covers " + strings.Join(c.Hosts, ", ") + ".", Href: href})
}
}
}
return out
}
func (srv *Server) handleCloudflare(w http.ResponseWriter, r *http.Request) {
if srv.cf == nil {
writeJSON(w, http.StatusOK, map[string]any{"configured": false})
return
}
cfg := srv.cf.Config()
snap := srv.cf.Snapshot()
changes := map[string]map[string][]string{}
for _, z := range snap.Zones {
if added, removed := srv.dns.diff(z.Name, records(z.Records)); len(added)+len(removed) > 0 {
changes[z.Name] = map[string][]string{"added": nonNilStrings(added), "removed": nonNilStrings(removed)}
}
}
writeJSON(w, http.StatusOK, map[string]any{
"configured": true, "snapshot": snap, "dns_changes": changes,
"error_warn_pct": cfg.ErrorWarnPct, "error_critical_pct": cfg.ErrorCriticalPct,
"conditions": nonNilConds(srv.cloudflareConditions(time.Now())),
"ssl_mode": cfg.SSLMode, "ssl_mode_zones": cfg.SSLModeZones, "cert_warn_days": cfg.CertWarnDays, "cert_critical_days": cfg.CertCriticalDays,
"every_s": int(cfg.Every / time.Second),
})
}
func nonNilConds(c []Condition) []Condition {
if c == nil {
return []Condition{}
}
return c
}
func (srv *Server) handleDNSAccept(w http.ResponseWriter, r *http.Request) {
if srv.cf == nil {
http.NotFound(w, r)
return
}
zone := r.PathValue("zone")
for _, z := range srv.cf.Snapshot().Zones {
if z.Name != zone {
continue
}
added, removed := srv.dns.accept(zone, records(z.Records))
who := "someone"
if _, s, ok := srv.sessions.get(r); ok {
who = s.User
}
srv.audit(who, "accepted DNS changes", fmt.Sprintf("%s: %d added, %d removed", zone, added, removed))
writeJSON(w, http.StatusOK, map[string]int{"added": added, "removed": removed})
return
}
http.NotFound(w, r)
}
// dnsBaseline holds each zone's DNS records as last accepted, by Record.Key.
// A zone seen for the first time is taken as it is. With a database the
// baseline survives restarts, so a change made while obs-beacond was down is
// still caught.
// Akamai's Edge DNS zones have one of their own (prefix "ak_dns:").
type dnsBaseline struct {
mu sync.Mutex
m map[string]map[string]string
db *state.DB
prefix string // the database key's; "cf_dns:" when empty
}
// records are Cloudflare's records as a baseline holds them: each one's
// whole setting to how it reads; nil when they were never read.
func records(recs []cloudflare.Record) map[string]string {
if recs == nil {
return nil
}
m := make(map[string]string, len(recs))
for _, r := range recs {
m[r.Key()] = r.String()
}
return m
}
func (b *dnsBaseline) key(zone string) string {
if b.prefix == "" {
return "cf_dns:" + zone
}
return b.prefix + zone
}
// base returns the zone's baseline, loading it or starting it from recs.
// Callers hold b.mu.
func (b *dnsBaseline) base(zone string, recs map[string]string) map[string]string {
if b.m == nil {
b.m = map[string]map[string]string{}
}
if m, ok := b.m[zone]; ok {
return m
}
if b.db != nil {
if v := b.db.Meta(b.key(zone)); v != "" {
var m map[string]string
if json.Unmarshal([]byte(v), &m) == nil {
b.m[zone] = m
return m
}
}
}
b.set(zone, recs)
return recs
}
func (b *dnsBaseline) set(zone string, m map[string]string) {
b.m[zone] = m
if b.db != nil {
v, _ := json.Marshal(m)
if err := b.db.SetMeta(b.key(zone), string(v)); err != nil {
log.Printf("can't store the DNS baseline for %s: %v", zone, err)
}
}
}
// diff lists what was added and removed since the baseline, readable and sorted.
func (b *dnsBaseline) diff(zone string, cur map[string]string) (added, removed []string) {
if cur == nil {
return nil, nil // never read: nothing to compare
}
b.mu.Lock()
defer b.mu.Unlock()
base := b.base(zone, cur)
for k, v := range cur {
if _, ok := base[k]; !ok {
added = append(added, v)
}
}
for k, v := range base {
if _, ok := cur[k]; !ok {
removed = append(removed, v)
}
}
sort.Strings(added)
sort.Strings(removed)
return added, removed
}
// accept makes the current records the baseline and says what that took in.
func (b *dnsBaseline) accept(zone string, cur map[string]string) (added, removed int) {
if cur == nil {
return 0, 0 // never read: nothing to accept
}
a, r := b.diff(zone, cur)
b.mu.Lock()
defer b.mu.Unlock()
b.set(zone, cur)
return len(a), len(r)
}
func nonNilStrings(s []string) []string {
if s == nil {
return []string{}
}
return s
}