Files
cairnobs/internal/cloudflare/cloudflare.go
T
jcoffey-dev 8d0b907f5f fleet: a fake Cloudflare account for the demo, through cloudflare's stand_in
cloudflare: stand_in takes a stand-in on this machine (http on loopback,
nothing else), which the token, every request and the status page go to.

fleet run -cloudflare DIR serves twelve zones and a load-balanced pool
per site whose origins are the sites' load balancers, with traffic for
the last 15 minutes, and writes the cloudflare: section to DIR. Wrong on
purpose: one zone on Flexible SSL, an edge certificate ending in 9 days,
an advanced certificate stuck validating, fleet-0052 failing its health
check from eastern North America, and 6% of shop.example's requests
failing on its checkout host.
2026-11-14 20:00:17 -07:00

689 lines
21 KiB
Go

// Package cloudflare reads what Cloudflare knows about our sites with a
// read-only API token: load balancer pools and their origins' health, each
// zone's SSL mode and edge certificates, and Cloudflare's own status page.
// It only reads; obs-beacond turns what it finds into conditions.
package cloudflare
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"slices"
"sort"
"strings"
"sync"
"time"
)
// Config is the `cloudflare` section of obs-beacond.yaml.
type Config struct {
TokenFile string `yaml:"token_file"` // read-only API token, re-read on every poll
Every time.Duration `yaml:"every"` // default 5m
Zones []string `yaml:"zones"` // empty: every zone the token can read
SSLMode string `yaml:"ssl_mode"` // what every zone should use; default strict
SSLModeZones map[string]string `yaml:"ssl_mode_zones"` // zones known to use another mode
CertWarnDays int `yaml:"cert_warn_days"` // default 14
CertCriticalDays int `yaml:"cert_critical_days"` // default 3
// Traffic over the last 15 minutes, from Cloudflare's request analytics.
ErrorWarnPct float64 `yaml:"error_warn_pct"` // share of requests answered 5xx; default 5
ErrorCriticalPct float64 `yaml:"error_critical_pct"` // default 25
ErrorMin int `yaml:"error_min"` // fewer 5xx than this never alerts; default 20
BlockMin int `yaml:"block_min"` // blocked requests before it can be a spike; default 100
BlockFactor float64 `yaml:"block_factor"` // times the last day's average; default 10
// StandIn is a stand-in for Cloudflare on this machine, such as the
// fleet simulator's (scripts/fleet): http on loopback only. The token
// and every request go there instead, the status page too.
StandIn string `yaml:"stand_in"`
API string `yaml:"-"` // tests point these at a fake
StatusURL string `yaml:"-"`
}
// WantSSL is the SSL mode zone should use.
func (c *Config) WantSSL(zone string) string {
if m, ok := c.SSLModeZones[zone]; ok {
return m
}
return c.SSLMode
}
// Defaults fills in what was left out.
func (c *Config) Defaults() {
if c.Every == 0 {
c.Every = 5 * time.Minute
}
if c.SSLMode == "" {
c.SSLMode = "strict"
}
if c.CertWarnDays == 0 {
c.CertWarnDays = 14
}
if c.CertCriticalDays == 0 {
c.CertCriticalDays = 3
}
if c.ErrorWarnPct == 0 {
c.ErrorWarnPct = 5
}
if c.ErrorCriticalPct == 0 {
c.ErrorCriticalPct = 25
}
if c.ErrorMin == 0 {
c.ErrorMin = 20
}
if c.BlockMin == 0 {
c.BlockMin = 100
}
if c.BlockFactor == 0 {
c.BlockFactor = 10
}
if c.API == "" {
c.API = "https://api.cloudflare.com/client/v4"
}
if c.StatusURL == "" {
c.StatusURL = "https://www.cloudflarestatus.com/api/v2/status.json"
}
}
// Snapshot is the latest reading. Lists are never nil.
type Snapshot struct {
Checked time.Time `json:"checked"` // last successful poll; zero before the first
Error string `json:"error,omitempty"` // the last poll's error, if it failed
ErrorSince time.Time `json:"error_since,omitzero"` // when polls started failing
TrafficError string `json:"traffic_error,omitempty"` // analytics failed; everything else is current
Zones []Zone `json:"zones"`
Pools []Pool `json:"pools"`
Status Status `json:"status"`
}
type Zone struct {
Name string `json:"name"`
SSL string `json:"ssl"` // off, flexible, full, strict
Certs []Cert `json:"certs"`
LBs []string `json:"load_balancers"` // hostnames with a load balancer
Traffic *Traffic `json:"traffic,omitempty"`
Records []Record `json:"-"` // obs-beacond compares these with what was last accepted
DNS int `json:"dns_records"`
}
// Traffic is one zone's requests over a 15-minute window ending a minute
// ago (Cloudflare's analytics lag about that much).
type Traffic struct {
Requests int `json:"requests"`
Errors int `json:"errors"` // answered 5xx
ErrorHosts []HostCount `json:"error_hosts"` // where the 5xx were, most first
Blocked int `json:"blocked"` // blocked or challenged by Cloudflare itself
BlockedDay int `json:"blocked_day"` // the 24 hours before the window
}
type HostCount struct {
Host string `json:"host"`
Count int `json:"count"`
}
// Record is a DNS record as far as drift is concerned: ids, comments and
// timestamps don't count.
type Record struct {
Type string `json:"type"`
Name string `json:"name"`
Content string `json:"content"`
Proxied bool `json:"proxied"`
TTL int `json:"ttl"`
Priority *int `json:"priority,omitempty"`
}
// Key identifies the record's whole setting.
func (r Record) Key() string {
p := ""
if r.Priority != nil {
p = fmt.Sprint(*r.Priority)
}
return fmt.Sprintf("%s %s %s proxied=%v ttl=%d prio=%s", r.Type, r.Name, r.Content, r.Proxied, r.TTL, p)
}
// String is how a person reads it.
func (r Record) String() string {
s := r.Type + " " + r.Name + " → " + r.Content
if r.Priority != nil {
s += fmt.Sprintf(" (priority %d)", *r.Priority)
}
if r.Proxied {
s += " (proxied)"
} else if r.TTL > 1 {
s += fmt.Sprintf(" (TTL %d)", r.TTL)
}
return s
}
type Cert struct {
Type string `json:"type"`
Status string `json:"status"`
Hosts []string `json:"hosts"`
Expires time.Time `json:"expires,omitzero"` // the soonest of the pack's certificates
}
type Pool struct {
Name string `json:"name"`
Enabled bool `json:"enabled"`
Healthy bool `json:"healthy"`
Origins []Origin `json:"origins"`
}
type Origin struct {
Name string `json:"name"`
Address string `json:"address"`
Enabled bool `json:"enabled"`
Healthy bool `json:"healthy"` // healthy from every region that checks it
Reason string `json:"reason,omitempty"`
Code int `json:"code,omitempty"`
RTT string `json:"rtt,omitempty"`
}
// Status is Cloudflare's own status page: indicator none, minor, major or critical.
type Status struct {
Indicator string `json:"indicator"`
Description string `json:"description"`
}
// Watcher polls Cloudflare and keeps the latest Snapshot.
type Watcher struct {
cfg Config
client *http.Client
mu sync.Mutex
snap Snapshot
}
// New checks the token file is readable; it is read again on every poll so
// a new token needs no restart.
func New(cfg Config) (*Watcher, error) {
if cfg.StandIn != "" {
u, err := url.Parse(cfg.StandIn)
if err != nil || u.Scheme != "http" || u.Path != "" && u.Path != "/" || !loopback(u.Hostname()) {
return nil, fmt.Errorf("cloudflare: stand_in %q: must be http://127.0.0.1:port (a stand-in on this machine)", cfg.StandIn)
}
cfg.API = "http://" + u.Host
cfg.StatusURL = cfg.API + "/status.json"
}
cfg.Defaults()
if cfg.TokenFile == "" {
return nil, errors.New("cloudflare: token_file is required")
}
if _, err := readToken(cfg.TokenFile); err != nil {
return nil, err
}
return &Watcher{cfg: cfg, client: &http.Client{Timeout: 20 * time.Second},
snap: Snapshot{Zones: []Zone{}, Pools: []Pool{}}}, nil
}
func loopback(host string) bool {
ip := net.ParseIP(host)
return host == "localhost" || ip != nil && ip.IsLoopback()
}
// Config returns the settings in use (defaults filled in).
func (w *Watcher) Config() Config { return w.cfg }
func readToken(p string) (string, error) {
b, err := os.ReadFile(p)
if err != nil {
return "", fmt.Errorf("cloudflare token: %w", err)
}
t := strings.TrimSpace(string(b))
if t == "" {
return "", fmt.Errorf("cloudflare token %s is empty", p)
}
return t, nil
}
// Snapshot returns the latest reading.
func (w *Watcher) Snapshot() Snapshot {
w.mu.Lock()
defer w.mu.Unlock()
return w.snap
}
// Set replaces the reading; tests use it to judge a known state.
func (w *Watcher) Set(s Snapshot) {
w.mu.Lock()
defer w.mu.Unlock()
w.snap = s
}
// Run polls now and then every cfg.Every until ctx is done.
func (w *Watcher) Run(ctx context.Context) {
t := time.NewTicker(w.cfg.Every)
defer t.Stop()
for {
w.Poll(ctx, time.Now())
select {
case <-ctx.Done():
return
case <-t.C:
}
}
}
// Poll reads everything once. On failure the previous reading is kept and
// the error recorded, so a short API outage doesn't blank the page.
func (w *Watcher) Poll(ctx context.Context, now time.Time) {
s, err := w.read(ctx)
w.mu.Lock()
defer w.mu.Unlock()
if err != nil {
if w.snap.Error == "" {
w.snap.ErrorSince = now
}
w.snap.Error = err.Error()
return
}
s.Checked = now
w.snap = s
}
// envelope is the API's wrapper around every answer.
type envelope struct {
Success bool `json:"success"`
Errors []apiError `json:"errors"`
Result json.RawMessage `json:"result"`
Info *struct {
Page int `json:"page"`
TotalPages int `json:"total_pages"`
} `json:"result_info"`
}
type apiError struct {
Code int `json:"code"`
Message string `json:"message"`
}
func (w *Watcher) get(ctx context.Context, token, path string) (envelope, error) {
var env envelope
req, err := http.NewRequestWithContext(ctx, http.MethodGet, w.cfg.API+path, nil)
if err != nil {
return env, err
}
req.Header.Set("Authorization", "Bearer "+token)
resp, err := w.client.Do(req)
if err != nil {
// The URL is ours and holds no secret; the token is only in a header.
return env, fmt.Errorf("cloudflare %s: %w", path, errors.Unwrap(err))
}
defer resp.Body.Close()
body, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
if err := json.Unmarshal(body, &env); err != nil {
return env, fmt.Errorf("cloudflare %s: HTTP %d, unreadable answer", path, resp.StatusCode)
}
if !env.Success {
msg := fmt.Sprintf("HTTP %d", resp.StatusCode)
if len(env.Errors) > 0 {
msg = fmt.Sprintf("%s (code %d)", env.Errors[0].Message, env.Errors[0].Code)
}
return env, fmt.Errorf("cloudflare %s: %s", path, msg)
}
return env, nil
}
// list follows the API's pages and decodes every item into out.
func list[T any](ctx context.Context, w *Watcher, token, path string) ([]T, error) {
var all []T
sep := "?"
if strings.Contains(path, "?") {
sep = "&"
}
for page := 1; ; page++ {
env, err := w.get(ctx, token, fmt.Sprintf("%s%spage=%d&per_page=50", path, sep, page))
if err != nil {
return nil, err
}
var items []T
if err := json.Unmarshal(env.Result, &items); err != nil {
return nil, fmt.Errorf("cloudflare %s: %w", path, err)
}
all = append(all, items...)
if env.Info == nil || page >= env.Info.TotalPages {
return all, nil
}
}
}
func (w *Watcher) read(ctx context.Context) (Snapshot, error) {
s := Snapshot{Zones: []Zone{}, Pools: []Pool{}}
token, err := readToken(w.cfg.TokenFile)
if err != nil {
return s, err
}
zones, err := list[zoneRow](ctx, w, token, "/zones")
if err != nil {
return s, err
}
want := map[string]bool{}
for _, z := range w.cfg.Zones {
want[strings.ToLower(z)] = true
}
accounts := map[string]bool{}
ids := map[string]int{} // zone id → index in s.Zones
// Zones are read six at a time: one after another, 26 zones took a
// minute, which left the page empty that long after every restart.
var todo []zoneRow
for _, z := range zones {
if len(want) > 0 && !want[z.Name] {
continue
}
accounts[z.Account.ID] = true
todo = append(todo, z)
}
read := make([]Zone, len(todo))
errs := make([]error, len(todo))
sem := make(chan struct{}, 6)
var wg sync.WaitGroup
for i, z := range todo {
wg.Add(1)
go func() {
defer wg.Done()
sem <- struct{}{}
defer func() { <-sem }()
read[i], errs[i] = w.readZone(ctx, token, z)
}()
}
wg.Wait()
for i, z := range todo {
if errs[i] != nil {
return s, errs[i]
}
s.Zones = append(s.Zones, read[i])
ids[z.ID] = len(s.Zones) - 1
}
if err := w.traffic(ctx, token, s.Zones, ids, time.Now()); err != nil {
s.TrafficError = err.Error()
}
for acct := range accounts {
type poolRow struct {
ID string `json:"id"`
Name string `json:"name"`
Enabled bool `json:"enabled"`
Healthy *bool `json:"healthy"`
Origins []struct {
Name string `json:"name"`
Address string `json:"address"`
Enabled bool `json:"enabled"`
} `json:"origins"`
}
pools, err := list[poolRow](ctx, w, token, "/accounts/"+acct+"/load_balancers/pools")
if err != nil {
return s, err
}
for _, p := range pools {
pool := Pool{Name: p.Name, Enabled: p.Enabled, Healthy: p.Healthy == nil || *p.Healthy, Origins: []Origin{}}
health, err := w.poolHealth(ctx, token, acct, p.ID)
if err != nil {
return s, err
}
for _, o := range p.Origins {
origin := Origin{Name: o.Name, Address: o.Address, Enabled: o.Enabled, Healthy: true}
if h, ok := health[o.Address]; ok {
origin.Healthy, origin.Reason, origin.Code, origin.RTT = h.Healthy, h.Reason, h.Code, h.RTT
}
pool.Origins = append(pool.Origins, origin)
}
s.Pools = append(s.Pools, pool)
}
}
sort.Slice(s.Pools, func(i, j int) bool { return s.Pools[i].Name < s.Pools[j].Name })
sort.Slice(s.Zones, func(i, j int) bool { return s.Zones[i].Name < s.Zones[j].Name })
// The status page is public and separate: failing to read it isn't a
// reason to drop everything else.
s.Status = w.status(ctx)
return s, nil
}
// poolHealth folds every region's view of each origin into one: an origin
// is healthy only if every region that checks it says so.
func (w *Watcher) poolHealth(ctx context.Context, token, acct, pool string) (map[string]Origin, error) {
env, err := w.get(ctx, token, "/accounts/"+acct+"/load_balancers/pools/"+url.PathEscape(pool)+"/health")
if err != nil {
return nil, err
}
var h struct {
PopHealth map[string]struct {
Origins []map[string]struct {
Healthy bool `json:"healthy"`
RTT string `json:"rtt"`
FailureReason string `json:"failure_reason"`
ResponseCode int `json:"response_code"`
} `json:"origins"`
} `json:"pop_health"`
}
if err := json.Unmarshal(env.Result, &h); err != nil {
return nil, fmt.Errorf("cloudflare pool health: %w", err)
}
out := map[string]Origin{}
regions := make([]string, 0, len(h.PopHealth))
for r := range h.PopHealth {
regions = append(regions, r)
}
sort.Strings(regions)
for _, r := range regions {
for _, m := range h.PopHealth[r].Origins {
for addr, o := range m {
cur, seen := out[addr]
if !seen {
cur = Origin{Healthy: true, RTT: o.RTT, Code: o.ResponseCode}
}
if !o.Healthy && cur.Healthy {
cur.Healthy, cur.Code, cur.RTT = false, o.ResponseCode, o.RTT
cur.Reason = o.FailureReason + " (seen from " + r + ")"
}
out[addr] = cur
}
}
}
return out, nil
}
func (w *Watcher) status(ctx context.Context) Status {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, w.cfg.StatusURL, nil)
if err != nil {
return Status{Indicator: "unknown", Description: err.Error()}
}
resp, err := w.client.Do(req)
if err != nil {
return Status{Indicator: "unknown", Description: "status page unreachable"}
}
defer resp.Body.Close()
var v struct {
Status Status `json:"status"`
}
if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&v); err != nil || v.Status.Indicator == "" {
return Status{Indicator: "unknown", Description: "status page unreadable"}
}
return v.Status
}
// traffic reads every zone's last 15 minutes (and the day before, for the
// blocked baseline) from the GraphQL analytics, ten zones per query. Only
// request analytics: the firewall events dataset needs a paid plan, so
// "blocked" is any request Cloudflare's security refused itself (see
// blockingActions).
func (w *Watcher) traffic(ctx context.Context, token string, zones []Zone, ids map[string]int, now time.Time) error {
end := now.UTC().Truncate(time.Minute).Add(-time.Minute)
start, dayStart := end.Add(-15*time.Minute), end.Add(-15*time.Minute-24*time.Hour)
const q = `query($tags:[string!], $a:Time!, $b:Time!, $c:Time!, $blocking:[string!]) { viewer { zones(filter:{zoneTag_in:$tags}) { zoneTag
win: httpRequestsAdaptiveGroups(limit:5000, filter:{datetime_geq:$a, datetime_lt:$b}) { count dimensions { edgeResponseStatus securityAction clientRequestHTTPHost } }
day: httpRequestsAdaptiveGroups(limit:5000, filter:{datetime_geq:$c, datetime_lt:$a, securityAction_in:$blocking}) { count }
} } }`
tags := make([]string, 0, len(ids))
for id := range ids {
tags = append(tags, id)
}
sort.Strings(tags)
for i := 0; i < len(tags); i += 10 {
chunk := tags[i:min(i+10, len(tags))]
body, _ := json.Marshal(map[string]any{"query": q, "variables": map[string]any{
"tags": chunk, "a": start.Format(time.RFC3339), "b": end.Format(time.RFC3339), "c": dayStart.Format(time.RFC3339),
"blocking": blockingActions}})
req, err := http.NewRequestWithContext(ctx, http.MethodPost, w.cfg.API+"/graphql", strings.NewReader(string(body)))
if err != nil {
return err
}
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("Content-Type", "application/json")
resp, err := w.client.Do(req)
if err != nil {
return fmt.Errorf("cloudflare analytics: %w", errors.Unwrap(err))
}
var out struct {
Data struct {
Viewer struct {
Zones []struct {
ZoneTag string `json:"zoneTag"`
Win []struct {
Count int `json:"count"`
Dim struct {
Status int `json:"edgeResponseStatus"`
Action string `json:"securityAction"`
Host string `json:"clientRequestHTTPHost"`
} `json:"dimensions"`
} `json:"win"`
Day []struct {
Count int `json:"count"`
} `json:"day"`
} `json:"zones"`
} `json:"viewer"`
} `json:"data"`
Errors []struct {
Message string `json:"message"`
} `json:"errors"`
}
err = json.NewDecoder(io.LimitReader(resp.Body, 16<<20)).Decode(&out)
resp.Body.Close()
if err != nil {
return fmt.Errorf("cloudflare analytics: HTTP %d, unreadable answer", resp.StatusCode)
}
if len(out.Errors) > 0 {
return fmt.Errorf("cloudflare analytics: %s", out.Errors[0].Message)
}
for _, z := range out.Data.Viewer.Zones {
idx, ok := ids[z.ZoneTag]
if !ok {
continue
}
t := &Traffic{ErrorHosts: []HostCount{}}
hosts := map[string]int{}
for _, g := range z.Win {
t.Requests += g.Count
if g.Dim.Status >= 500 {
t.Errors += g.Count
hosts[g.Dim.Host] += g.Count
}
if blocking(g.Dim.Action) {
t.Blocked += g.Count
}
}
for _, g := range z.Day {
t.BlockedDay += g.Count
}
for h, n := range hosts {
t.ErrorHosts = append(t.ErrorHosts, HostCount{h, n})
}
sort.Slice(t.ErrorHosts, func(i, j int) bool { return t.ErrorHosts[i].Count > t.ErrorHosts[j].Count })
zones[idx].Traffic = t
}
}
return nil
}
// blockingActions are the security actions that refused a request at the
// edge: a block, a challenge page, or a closed connection. It is a list of
// what counts, not of what doesn't, because Cloudflare keeps adding actions
// that let the request through: AI Labyrinth's link_maze_injected (decoy links
// added to a page that was served), challenge_solved and the other *_solved
// and *_bypassed follow-ups, rewrite. Counting those turned a crawler being
// fed decoy links into "Cloudflare is blocking a lot of traffic".
var blockingActions = []string{
"block",
"challenge", "jschallenge", "managed_challenge",
"connection_close", "force_connection_close",
}
// blocking is a security action that answered the request at the edge.
func blocking(action string) bool {
return slices.Contains(blockingActions, action)
}
type zoneRow struct {
ID string `json:"id"`
Name string `json:"name"`
Account struct {
ID string `json:"id"`
} `json:"account"`
}
// readZone reads one zone's SSL mode, certificates, load balancers and DNS.
func (w *Watcher) readZone(ctx context.Context, token string, z zoneRow) (Zone, error) {
zone := Zone{Name: z.Name, Certs: []Cert{}, LBs: []string{}}
env, err := w.get(ctx, token, "/zones/"+z.ID+"/settings/ssl")
if err != nil {
return zone, err
}
var ssl struct {
Value string `json:"value"`
}
_ = json.Unmarshal(env.Result, &ssl)
zone.SSL = ssl.Value
type packRow struct {
Type string `json:"type"`
Status string `json:"status"`
Hosts []string `json:"hosts"`
Certificates []struct {
ExpiresOn time.Time `json:"expires_on"`
} `json:"certificates"`
}
packs, err := list[packRow](ctx, w, token, "/zones/"+z.ID+"/ssl/certificate_packs")
if err != nil {
return zone, err
}
for _, p := range packs {
c := Cert{Type: p.Type, Status: p.Status, Hosts: p.Hosts}
for _, x := range p.Certificates {
if !x.ExpiresOn.IsZero() && (c.Expires.IsZero() || x.ExpiresOn.Before(c.Expires)) {
c.Expires = x.ExpiresOn
}
}
if c.Hosts == nil {
c.Hosts = []string{}
}
zone.Certs = append(zone.Certs, c)
}
type lbRow struct {
Name string `json:"name"`
}
lbs, err := list[lbRow](ctx, w, token, "/zones/"+z.ID+"/load_balancers")
if err != nil {
return zone, err
}
for _, lb := range lbs {
zone.LBs = append(zone.LBs, lb.Name)
}
if zone.Records, err = list[Record](ctx, w, token, "/zones/"+z.ID+"/dns_records"); err != nil {
return zone, err
}
if zone.Records == nil {
zone.Records = []Record{}
}
zone.DNS = len(zone.Records)
return zone, nil
}