Files
cairnobs/internal/server/akamai.go
T
jcoffey-dev db156c4f06
release / release (push) Successful in 7m6s
Cairn OBS
The new Cairn OBS: a self-hosted monitor for networks, hosts and
services. Licensed AGPLv3.
2026-11-14 03:27:09 -07:00

316 lines
11 KiB
Go

package server
import (
"fmt"
"net"
"net/http"
"slices"
"strconv"
"strings"
"time"
"github.com/cairnobs/cairnobs/internal/akamai"
"github.com/cairnobs/cairnobs/internal/certinfo"
)
// akamaiPending is how long a GTM change may take to propagate before
// it's said to be stuck.
const akamaiPending = 30 * time.Minute
// akamaiConditions judges the latest Akamai reading (docs/akamai.md,
// Judging), as Cloudflare's are.
func (srv *Server) akamaiConditions(now time.Time) []Condition {
if srv.ak == nil {
return nil
}
s, cfg := srv.ak.Snapshot(), srv.ak.Config()
const href = "/akamai"
var out []Condition
if s.Error != "" && now.Sub(s.ErrorSince) >= 3*cfg.Every {
out = append(out, Condition{Key: "akamai/api", Severity: "warning", Title: "Can't read Akamai",
Detail: s.Error + ". Traffic management checks are on hold; the last reading stays on screen.", Href: href})
}
if s.Checked.IsZero() {
return out
}
hostOf := srv.hostsByAddr()
for _, d := range s.Domains {
switch {
case d.Propagation == "DENIED":
out = append(out, Condition{Key: "akamai/propagation/" + d.Name, Severity: "warning",
Title: "Akamai refused the last change to " + d.Name, Detail: strings.TrimSuffix(d.Message, ".") + ".", Href: href})
case d.Propagation == "PENDING" && !d.Since.IsZero() && now.Sub(d.Since) > akamaiPending:
out = append(out, Condition{Key: "akamai/propagation/" + d.Name, Severity: "warning",
Title: "A change to " + d.Name + " has been propagating for " + roughly(now.Sub(d.Since)),
Detail: "GTM changes usually reach every nameserver within minutes; until then some answer with the old configuration.", Href: href})
}
for _, p := range d.Properties {
if p.At.IsZero() {
continue // no availability report yet
}
name := p.Name + "." + d.Name
var down []string
enabled, live := 0, 0
for _, t := range p.Targets {
if !t.Enabled || len(t.Servers) == 0 {
continue
}
enabled++
if t.Live() {
live++
}
for _, sv := range t.Servers {
if !sv.Known || sv.Alive {
continue
}
down = append(down, t.Datacenter+" ("+sv.IP+")")
out = append(out, Condition{Key: "akamai/server/" + name + "/" + sv.IP, Severity: "warning",
Title: fmt.Sprintf("%s in %s is failing GTM's liveness test for %s", sv.IP, t.Datacenter, name),
Detail: "Akamai stops handing it out and sends its share to the other datacenters.", Host: hostOf[sv.IP], Href: href})
}
}
if enabled > 0 && live == 0 {
out = append(out, Condition{Key: "akamai/property/" + name, Severity: "critical",
Title: "Akamai GTM: " + name + " has no live datacenter",
Detail: "Every datacenter's servers are failing GTM's liveness tests: " + strings.Join(down, ", ") + ".", Href: href})
}
}
}
for _, e := range s.Enrollments {
key := "akamai/cps/" + strconv.Itoa(e.ID)
for _, c := range e.Deployed {
if st, word := certStatus(c, now); st != "good" {
out = append(out, Condition{Key: key + "/" + c.Fingerprint, Severity: st,
Title: "Akamai certificate for " + e.CN + ": " + lowerFirst(word),
Detail: akamaiEnrollment(e) + ", " + c.Key + ". " + akamaiRenewal(e), Href: href})
}
}
if what := akamaiWaiting[e.Waiting]; what != "" {
out = append(out, Condition{Key: key + "/waiting", Severity: "warning",
Title: "Akamai certificate for " + e.CN + ": the renewal is waiting on you",
Detail: "CPS needs " + what + " before it can go on (" + akamaiEnrollment(e) + ").", Href: href})
}
}
for _, p := range s.PMProperties {
for _, n := range []struct {
name string
a *akamai.Activation
now int
}{{"staging", p.LastStaging, p.Staging}, {"production", p.LastProduction, p.Production}} {
a := n.a
if a == nil {
continue
}
verb := "Activating"
if a.Type == "DEACTIVATE" {
verb = "Deactivating"
}
what := fmt.Sprintf("%s %s v%d on %s", verb, p.Name, a.Version, n.name)
stays := "nothing is active there"
if n.now > 0 {
stays = fmt.Sprintf("v%d stays active", n.now)
}
note := ""
if a.Note != "" {
note = " Its note: " + strings.TrimSuffix(a.Note, ".") + "."
}
key := "akamai/activation/" + p.ID + "/" + n.name
switch {
case a.Status == "FAILED":
out = append(out, Condition{Key: key, Severity: "warning", Title: "Akamai: " + lowerFirst(what) + " failed",
Detail: fmt.Sprintf("Submitted %s; %s.%s", a.Submitted.UTC().Format("2006-01-02 15:04 UTC"), stays, note), Href: href})
case a.Pending() && now.Sub(a.Submitted) > akamaiActivationPending:
out = append(out, Condition{Key: key, Severity: "warning", Title: "Akamai: " + lowerFirst(what) + " has been pending for " + roughly(now.Sub(a.Submitted)),
Detail: fmt.Sprintf("Activations usually finish well within an hour (it's at %s); until it does, %s.%s", a.Status, stays, note), Href: href})
}
}
}
for _, t := range s.Traffic {
if t.Errors < int64(cfg.ErrorMin) {
continue
}
sev := ""
switch pct := t.Pct(); {
case pct >= cfg.ErrorCriticalPct:
sev = "critical"
case pct >= cfg.ErrorWarnPct:
sev = "warning"
}
if sev != "" {
out = append(out, Condition{Key: fmt.Sprintf("akamai/traffic/%d", t.CPCode), Severity: sev,
Title: fmt.Sprintf("Akamai: %.0f%% of CP code %d's requests are failing with 5xx", t.Pct(), t.CPCode),
Detail: fmt.Sprintf("%d of %d edge hits between %s and %s UTC (the latest 5 minutes Akamai has reported). Usually the origin behind it; the GTM and Property Manager panels say whether Akamai changed anything.",
t.Errors, t.Hits, s.TrafficStart.UTC().Format("15:04"), s.TrafficEnd.UTC().Format("15:04")), Href: href})
}
}
for _, z := range s.DNSZones {
if added, removed := srv.akdns.diff(z.Name, akamaiRecords(z.Records)); len(added)+len(removed) > 0 {
var lines []string
for _, r := range removed {
lines = append(lines, "removed "+r)
}
for _, r := range added {
lines = append(lines, "added "+r)
}
if len(lines) > 4 {
lines = append(lines[:3], fmt.Sprintf("and %d more", len(lines)-3))
}
out = append(out, Condition{Key: "akamai/dns/" + z.Name, Severity: "warning",
Title: fmt.Sprintf("Akamai Edge DNS for %s changed (%d added, %d removed)", z.Name, len(added), len(removed)),
Detail: strings.Join(lines, "; ") + ". Accept it on the Akamai page if it was meant.", Href: href})
}
}
return out
}
// akamaiRecords are a zone's record sets as a DNS baseline holds them;
// nil when they weren't read.
func akamaiRecords(recs []akamai.RecordSet) map[string]string {
if recs == nil {
return nil
}
m := make(map[string]string, len(recs))
for _, r := range recs {
m[r.Key()] = r.String()
}
return m
}
func (srv *Server) handleAkamaiDNSAccept(w http.ResponseWriter, r *http.Request) {
if srv.ak == nil {
http.NotFound(w, r)
return
}
zone := r.PathValue("zone")
for _, z := range srv.ak.Snapshot().DNSZones {
if z.Name != zone || z.Records == nil {
continue
}
added, removed := srv.akdns.accept(zone, akamaiRecords(z.Records))
who := "someone"
if _, s, ok := srv.sessions.get(r); ok {
who = s.User
}
srv.audit(who, "accepted Akamai DNS changes", fmt.Sprintf("%s: %d added, %d removed", zone, added, removed))
writeJSON(w, http.StatusOK, map[string]int{"added": added, "removed": removed})
return
}
http.NotFound(w, r)
}
// akamaiActivationPending is how long a Property Manager activation may
// take before it's said to be stuck.
const akamaiActivationPending = time.Hour
// akamaiWaiting says what each kind of input a CPS change can be waiting
// for is, in words.
var akamaiWaiting = map[string]string{
"third-party-csr": "the CSR signed by your certificate authority and uploaded",
"third-party-certificate": "the certificate your authority issued, uploaded",
"change-management-info": "the certificate on staging checked and acknowledged",
"lets-encrypt-challenges": "the domain validation challenges answered",
"pre-verification-warnings-acknowledgement": "its warnings acknowledged",
"post-verification-warnings-acknowledgement": "its warnings acknowledged",
}
// akamaiEnrollment names an enrollment: "CPS enrollment 1002 (shop.example.com)".
func akamaiEnrollment(e akamai.Enrollment) string {
return "CPS enrollment " + strconv.Itoa(e.ID) + " (" + e.CN + ")"
}
// akamaiRenewal says who renews it.
func akamaiRenewal(e akamai.Enrollment) string {
switch {
case e.Validation == "third-party":
return "A third-party certificate: renewing it is up to you, in Control Center → Certificate Provisioning System."
case e.Pending > 0:
return "A renewal is in progress in CPS."
}
return "CPS starts renewing its own certificates well before they end, so one this close means a renewal stalled: look in Control Center → Certificate Provisioning System."
}
// roughly is a duration in words: "45 minutes", "3 hours".
func roughly(d time.Duration) string {
if d < 2*time.Hour {
return plural(int(d.Minutes()), "minute", "minutes")
}
return plural(int(d.Hours()), "hour", "hours")
}
// hostsByAddr maps each address a host's map sample has on an interface
// to the host, so a server Akamai names is the host it is.
func (srv *Server) hostsByAddr() map[string]string {
m := map[string]string{}
for _, h := range srv.store.Snapshot() {
if h.mp == nil {
continue
}
for _, i := range h.mp.Interfaces {
for _, a := range i.Addrs {
if ip, _, err := net.ParseCIDR(a); err == nil {
m[ip.String()] = h.name
} else if ip := net.ParseIP(a); ip != nil {
m[ip.String()] = h.name
}
}
}
}
return m
}
func (srv *Server) handleAkamai(w http.ResponseWriter, r *http.Request) {
if srv.ak == nil {
writeJSON(w, http.StatusOK, map[string]any{"configured": false})
return
}
snap, cfg := srv.ak.Snapshot(), srv.ak.Config()
hostOf := srv.hostsByAddr()
hosts := map[string]string{}
for _, d := range snap.Domains {
for _, p := range d.Properties {
for _, t := range p.Targets {
for _, s := range t.Servers {
if h := hostOf[s.IP]; h != "" {
hosts[s.IP] = h
}
}
}
}
}
// Each deployed certificate judged as on the Certificates page.
type judged struct {
Status string `json:"status"`
Word string `json:"word"`
Weak string `json:"weak,omitempty"`
}
certs := map[string]judged{}
now := time.Now()
for _, e := range snap.Enrollments {
for _, c := range e.Deployed {
st, word := certStatus(c, now)
certs[c.Fingerprint] = judged{st, word, certinfo.Weak(&c)}
}
}
changes := map[string]map[string][]string{}
for _, z := range snap.DNSZones {
if added, removed := srv.akdns.diff(z.Name, akamaiRecords(z.Records)); len(added)+len(removed) > 0 {
changes[z.Name] = map[string][]string{"added": nonNilStrings(added), "removed": nonNilStrings(removed)}
}
}
// Record sets aren't sent: thousands, and the changes are what matters.
// (A copy: the watcher's reading shares its slices.)
snap.DNSZones = slices.Clone(snap.DNSZones)
for i := range snap.DNSZones {
snap.DNSZones[i].Count = len(snap.DNSZones[i].Records)
snap.DNSZones[i].Records = nil
}
writeJSON(w, http.StatusOK, map[string]any{
"configured": true, "snapshot": snap, "hosts": hosts, "certs": certs, "dns_changes": changes, "every_s": int(cfg.Every / time.Second),
"error_warn_pct": cfg.ErrorWarnPct, "error_critical_pct": cfg.ErrorCriticalPct, "error_min": cfg.ErrorMin,
"conditions": nonNilConds(srv.akamaiConditions(now)),
})
}
// akamaiReading is how the evaluator's reading is shared.
type akamaiReading = akamai.Snapshot