release / release (push) Successful in 7m6s
The new Cairn OBS: a self-hosted monitor for networks, hosts and services. Licensed AGPLv3.
280 lines
9.3 KiB
Go
280 lines
9.3 KiB
Go
package server
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"log"
|
|
"net/http"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/cairnobs/cairnobs/internal/cloudflare"
|
|
"github.com/cairnobs/cairnobs/internal/state"
|
|
)
|
|
|
|
// cloudflareConditions judges the latest Cloudflare reading. The status
|
|
// page is shown but never alerts: a Cloudflare incident somewhere in the
|
|
// world usually isn't ours, and when it is, the pools say so.
|
|
func (srv *Server) cloudflareConditions(now time.Time) []Condition {
|
|
if srv.cf == nil {
|
|
return nil
|
|
}
|
|
s, cfg := srv.cf.Snapshot(), srv.cf.Config()
|
|
const href = "/cloudflare"
|
|
var out []Condition
|
|
if s.Error != "" && now.Sub(s.ErrorSince) >= 3*cfg.Every {
|
|
out = append(out, Condition{Key: "cloudflare/api", Severity: "warning", Title: "Can't read Cloudflare",
|
|
Detail: s.Error + ". Load balancer, SSL and certificate checks are on hold.", Href: href})
|
|
}
|
|
if s.Checked.IsZero() {
|
|
return out
|
|
}
|
|
for _, p := range s.Pools {
|
|
if !p.Enabled {
|
|
continue
|
|
}
|
|
if !p.Healthy {
|
|
out = append(out, Condition{Key: "cloudflare/pool/" + p.Name, Severity: "critical",
|
|
Title: "Cloudflare pool " + p.Name + " has no healthy origin",
|
|
Detail: "Every origin is failing Cloudflare's health check, so the sites behind it are down or on a fallback.", Href: href})
|
|
}
|
|
for _, o := range p.Origins {
|
|
if !o.Enabled || o.Healthy {
|
|
continue
|
|
}
|
|
detail := o.Reason
|
|
if o.Code != 0 {
|
|
detail += fmt.Sprintf(", HTTP %d", o.Code)
|
|
}
|
|
out = append(out, Condition{Key: "cloudflare/origin/" + p.Name + "/" + o.Name, Severity: "warning",
|
|
Title: fmt.Sprintf("%s is failing %s's health check", o.Name, p.Name),
|
|
Detail: detail + ". Cloudflare sends its share of traffic to the other origins.", Host: o.Name, Href: href})
|
|
}
|
|
}
|
|
for _, z := range s.Zones {
|
|
if t := z.Traffic; t != nil && t.Requests > 0 {
|
|
pct := 100 * float64(t.Errors) / float64(t.Requests)
|
|
sev := ""
|
|
switch {
|
|
case t.Errors >= cfg.ErrorMin && pct >= cfg.ErrorCriticalPct:
|
|
sev = "critical"
|
|
case t.Errors >= cfg.ErrorMin && pct >= cfg.ErrorWarnPct:
|
|
sev = "warning"
|
|
}
|
|
if sev != "" {
|
|
detail := fmt.Sprintf("%d of %d requests in the last 15 minutes were answered with a server error", t.Errors, t.Requests)
|
|
if len(t.ErrorHosts) > 0 {
|
|
detail += fmt.Sprintf(", most on %s (%d)", t.ErrorHosts[0].Host, t.ErrorHosts[0].Count)
|
|
}
|
|
out = append(out, Condition{Key: "cloudflare/errors/" + z.Name, Severity: sev,
|
|
Title: fmt.Sprintf("%s: %.0f%% of requests failing", z.Name, pct), Detail: detail + ".", Href: href})
|
|
}
|
|
usual := float64(t.BlockedDay) / 96 // the day before, per 15 minutes
|
|
if t.Blocked >= cfg.BlockMin && float64(t.Blocked) >= cfg.BlockFactor*usual {
|
|
out = append(out, Condition{Key: "cloudflare/blocks/" + z.Name, Severity: "warning",
|
|
Title: fmt.Sprintf("Cloudflare is blocking a lot of %s traffic", z.Name),
|
|
Detail: fmt.Sprintf("%d requests blocked or challenged in the last 15 minutes; usually about %.0f. An attack, a bot, or a rule catching real visitors.", t.Blocked, usual), Href: href})
|
|
}
|
|
}
|
|
if added, removed := srv.dns.diff(z.Name, records(z.Records)); len(added)+len(removed) > 0 {
|
|
var lines []string
|
|
for _, r := range removed {
|
|
lines = append(lines, "removed "+r)
|
|
}
|
|
for _, r := range added {
|
|
lines = append(lines, "added "+r)
|
|
}
|
|
if len(lines) > 4 {
|
|
lines = append(lines[:3], fmt.Sprintf("and %d more", len(lines)-3))
|
|
}
|
|
out = append(out, Condition{Key: "cloudflare/dns/" + z.Name, Severity: "warning",
|
|
Title: fmt.Sprintf("DNS for %s changed (%d added, %d removed)", z.Name, len(added), len(removed)),
|
|
Detail: strings.Join(lines, "; ") + ". Accept it on the Cloudflare page if it was meant.", Href: href})
|
|
}
|
|
if want := cfg.WantSSL(z.Name); z.SSL != "" && z.SSL != want {
|
|
out = append(out, Condition{Key: "cloudflare/ssl/" + z.Name, Severity: "warning",
|
|
Title: fmt.Sprintf("%s SSL mode is %s, not %s", z.Name, z.SSL, want),
|
|
Detail: "Cloudflare's SSL/TLS encryption mode for the zone changed.", Href: href})
|
|
}
|
|
for _, c := range z.Certs {
|
|
key := "cloudflare/cert/" + z.Name + "/" + strings.Join(c.Hosts, ",")
|
|
days := int(c.Expires.Sub(now).Hours() / 24)
|
|
switch {
|
|
case c.Status != "active":
|
|
out = append(out, Condition{Key: key, Severity: "warning",
|
|
Title: fmt.Sprintf("%s edge certificate is %s", z.Name, strings.ReplaceAll(c.Status, "_", " ")),
|
|
Detail: "Covers " + strings.Join(c.Hosts, ", ") + ".", Href: href})
|
|
case c.Expires.IsZero():
|
|
case days < cfg.CertCriticalDays:
|
|
out = append(out, Condition{Key: key, Severity: "critical",
|
|
Title: fmt.Sprintf("%s edge certificate expires in %s", z.Name, plural(days, "day", "days")),
|
|
Detail: "Cloudflare normally renews well before this; renewal is stuck. Covers " + strings.Join(c.Hosts, ", ") + ".", Href: href})
|
|
case days < cfg.CertWarnDays:
|
|
out = append(out, Condition{Key: key, Severity: "warning",
|
|
Title: fmt.Sprintf("%s edge certificate expires in %s", z.Name, plural(days, "day", "days")),
|
|
Detail: "Cloudflare normally renews 30 days ahead. Covers " + strings.Join(c.Hosts, ", ") + ".", Href: href})
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (srv *Server) handleCloudflare(w http.ResponseWriter, r *http.Request) {
|
|
if srv.cf == nil {
|
|
writeJSON(w, http.StatusOK, map[string]any{"configured": false})
|
|
return
|
|
}
|
|
cfg := srv.cf.Config()
|
|
snap := srv.cf.Snapshot()
|
|
changes := map[string]map[string][]string{}
|
|
for _, z := range snap.Zones {
|
|
if added, removed := srv.dns.diff(z.Name, records(z.Records)); len(added)+len(removed) > 0 {
|
|
changes[z.Name] = map[string][]string{"added": nonNilStrings(added), "removed": nonNilStrings(removed)}
|
|
}
|
|
}
|
|
writeJSON(w, http.StatusOK, map[string]any{
|
|
"configured": true, "snapshot": snap, "dns_changes": changes,
|
|
"error_warn_pct": cfg.ErrorWarnPct, "error_critical_pct": cfg.ErrorCriticalPct,
|
|
"conditions": nonNilConds(srv.cloudflareConditions(time.Now())),
|
|
"ssl_mode": cfg.SSLMode, "ssl_mode_zones": cfg.SSLModeZones, "cert_warn_days": cfg.CertWarnDays, "cert_critical_days": cfg.CertCriticalDays,
|
|
"every_s": int(cfg.Every / time.Second),
|
|
})
|
|
}
|
|
|
|
func nonNilConds(c []Condition) []Condition {
|
|
if c == nil {
|
|
return []Condition{}
|
|
}
|
|
return c
|
|
}
|
|
|
|
func (srv *Server) handleDNSAccept(w http.ResponseWriter, r *http.Request) {
|
|
if srv.cf == nil {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
zone := r.PathValue("zone")
|
|
for _, z := range srv.cf.Snapshot().Zones {
|
|
if z.Name != zone {
|
|
continue
|
|
}
|
|
added, removed := srv.dns.accept(zone, records(z.Records))
|
|
who := "someone"
|
|
if _, s, ok := srv.sessions.get(r); ok {
|
|
who = s.User
|
|
}
|
|
srv.audit(who, "accepted DNS changes", fmt.Sprintf("%s: %d added, %d removed", zone, added, removed))
|
|
writeJSON(w, http.StatusOK, map[string]int{"added": added, "removed": removed})
|
|
return
|
|
}
|
|
http.NotFound(w, r)
|
|
}
|
|
|
|
// dnsBaseline holds each zone's DNS records as last accepted, by Record.Key.
|
|
// A zone seen for the first time is taken as it is. With a database the
|
|
// baseline survives restarts, so a change made while obs-beacond was down is
|
|
// still caught.
|
|
// Akamai's Edge DNS zones have one of their own (prefix "ak_dns:").
|
|
type dnsBaseline struct {
|
|
mu sync.Mutex
|
|
m map[string]map[string]string
|
|
db *state.DB
|
|
prefix string // the database key's; "cf_dns:" when empty
|
|
}
|
|
|
|
// records are Cloudflare's records as a baseline holds them: each one's
|
|
// whole setting to how it reads; nil when they were never read.
|
|
func records(recs []cloudflare.Record) map[string]string {
|
|
if recs == nil {
|
|
return nil
|
|
}
|
|
m := make(map[string]string, len(recs))
|
|
for _, r := range recs {
|
|
m[r.Key()] = r.String()
|
|
}
|
|
return m
|
|
}
|
|
|
|
func (b *dnsBaseline) key(zone string) string {
|
|
if b.prefix == "" {
|
|
return "cf_dns:" + zone
|
|
}
|
|
return b.prefix + zone
|
|
}
|
|
|
|
// base returns the zone's baseline, loading it or starting it from recs.
|
|
// Callers hold b.mu.
|
|
func (b *dnsBaseline) base(zone string, recs map[string]string) map[string]string {
|
|
if b.m == nil {
|
|
b.m = map[string]map[string]string{}
|
|
}
|
|
if m, ok := b.m[zone]; ok {
|
|
return m
|
|
}
|
|
if b.db != nil {
|
|
if v := b.db.Meta(b.key(zone)); v != "" {
|
|
var m map[string]string
|
|
if json.Unmarshal([]byte(v), &m) == nil {
|
|
b.m[zone] = m
|
|
return m
|
|
}
|
|
}
|
|
}
|
|
b.set(zone, recs)
|
|
return recs
|
|
}
|
|
|
|
func (b *dnsBaseline) set(zone string, m map[string]string) {
|
|
b.m[zone] = m
|
|
if b.db != nil {
|
|
v, _ := json.Marshal(m)
|
|
if err := b.db.SetMeta(b.key(zone), string(v)); err != nil {
|
|
log.Printf("can't store the DNS baseline for %s: %v", zone, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// diff lists what was added and removed since the baseline, readable and sorted.
|
|
func (b *dnsBaseline) diff(zone string, cur map[string]string) (added, removed []string) {
|
|
if cur == nil {
|
|
return nil, nil // never read: nothing to compare
|
|
}
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
base := b.base(zone, cur)
|
|
for k, v := range cur {
|
|
if _, ok := base[k]; !ok {
|
|
added = append(added, v)
|
|
}
|
|
}
|
|
for k, v := range base {
|
|
if _, ok := cur[k]; !ok {
|
|
removed = append(removed, v)
|
|
}
|
|
}
|
|
sort.Strings(added)
|
|
sort.Strings(removed)
|
|
return added, removed
|
|
}
|
|
|
|
// accept makes the current records the baseline and says what that took in.
|
|
func (b *dnsBaseline) accept(zone string, cur map[string]string) (added, removed int) {
|
|
if cur == nil {
|
|
return 0, 0 // never read: nothing to accept
|
|
}
|
|
a, r := b.diff(zone, cur)
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
b.set(zone, cur)
|
|
return len(a), len(r)
|
|
}
|
|
|
|
func nonNilStrings(s []string) []string {
|
|
if s == nil {
|
|
return []string{}
|
|
}
|
|
return s
|
|
}
|