release / release (push) Successful in 7m6s
The new Cairn OBS: a self-hosted monitor for networks, hosts and services. Licensed AGPLv3.
481 lines
16 KiB
Go
481 lines
16 KiB
Go
// Package akamai reads what Akamai knows about our sites, with a read-only
|
|
// API client (docs/akamai.md): Global Traffic Management domains, their
|
|
// properties and each datacenter's servers, alive or not. It only reads
|
|
// (GET, EdgeGrid-signed); obs-beacond turns what it finds into conditions.
|
|
package akamai
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"slices"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// Config is the `akamai` section of obs-beacond.yaml.
|
|
type Config struct {
|
|
Edgerc string `yaml:"edgerc_file"` // the API client's .edgerc, re-read on every poll
|
|
Section string `yaml:"section"` // default "default"
|
|
Every time.Duration `yaml:"every"` // default 5m
|
|
Domains []string `yaml:"domains"` // GTM domains; empty: every one the client can read
|
|
// Contracts whose CPS enrollments are read; empty: every contract
|
|
// Property Manager lists, else none named.
|
|
Contracts []string `yaml:"contracts"`
|
|
|
|
// Traffic (step 5): a CP code's share of edge hits answered 5xx, as
|
|
// Cloudflare's: warning and critical percentages, and fewer 5xx than
|
|
// ErrorMin never alert. Defaults 5, 25, 20.
|
|
ErrorWarnPct float64 `yaml:"error_warn_pct"`
|
|
ErrorCriticalPct float64 `yaml:"error_critical_pct"`
|
|
ErrorMin int `yaml:"error_min"`
|
|
|
|
API string `yaml:"-"` // tests point this at a fake: "http://127.0.0.1:…"
|
|
StatusURL string `yaml:"-"` // Akamai's status page summary; tests point it at a fake
|
|
}
|
|
|
|
// Defaults fills in what was left out.
|
|
func (c *Config) Defaults() {
|
|
if c.Section == "" {
|
|
c.Section = "default"
|
|
}
|
|
if c.Every == 0 {
|
|
c.Every = 5 * time.Minute
|
|
}
|
|
if c.ErrorWarnPct == 0 {
|
|
c.ErrorWarnPct = 5
|
|
}
|
|
if c.ErrorCriticalPct == 0 {
|
|
c.ErrorCriticalPct = 25
|
|
}
|
|
if c.ErrorMin == 0 {
|
|
c.ErrorMin = 20
|
|
}
|
|
if c.StatusURL == "" && c.API != "" {
|
|
c.StatusURL = c.API + "/status.json" // a fake's
|
|
}
|
|
if c.StatusURL == "" {
|
|
c.StatusURL = "https://www.akamaistatus.com/api/v2/status.json"
|
|
}
|
|
}
|
|
|
|
// Snapshot is one reading.
|
|
type Snapshot struct {
|
|
Checked time.Time `json:"checked"` // the last good reading; zero before the first
|
|
Error string `json:"error,omitempty"`
|
|
ErrorSince time.Time `json:"error_since"`
|
|
Domains []Domain `json:"domains"`
|
|
// CPS enrollments (step 2); nil when CPS isn't read. Read every
|
|
// cpsEvery, not every poll: CPSAt is when they were.
|
|
Enrollments []Enrollment `json:"enrollments"`
|
|
CPSAt time.Time `json:"cps_at"`
|
|
CPSError string `json:"cps_error,omitempty"` // the last CPS reading failed; Enrollments are the one before
|
|
// Property Manager properties (step 3), on the same pacing as CPS.
|
|
PMProperties []PMProperty `json:"pm_properties"`
|
|
PAPIAt time.Time `json:"papi_at"`
|
|
PAPIError string `json:"papi_error,omitempty"`
|
|
// Traffic (step 5): each CP code's edge hits and 5xx over a 5-minute
|
|
// window, read every poll; TrafficError when the last read failed.
|
|
Traffic []CPTraffic `json:"traffic"`
|
|
TrafficStart time.Time `json:"traffic_start"`
|
|
TrafficEnd time.Time `json:"traffic_end"`
|
|
TrafficError string `json:"traffic_error,omitempty"`
|
|
// Akamai's status page: shown, never alerting.
|
|
Status Status `json:"status"`
|
|
// Edge DNS zones (step 4), on the same pacing.
|
|
DNSZones []DNSZone `json:"dns_zones"`
|
|
DNSAt time.Time `json:"dns_at"`
|
|
DNSError string `json:"dns_error,omitempty"`
|
|
Absent []string `json:"absent,omitempty"` // products the client can't read ("gtm"): not in the account, or not granted
|
|
}
|
|
|
|
// Domain is a GTM domain.
|
|
type Domain struct {
|
|
Name string `json:"name"`
|
|
Type string `json:"type,omitempty"` // basic, full, weighted, static, failover-only
|
|
Propagation string `json:"propagation,omitempty"` // COMPLETE, PENDING, DENIED
|
|
Since time.Time `json:"since"` // when the propagation status was set
|
|
Message string `json:"message,omitempty"`
|
|
Properties []Property `json:"properties"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
// Property is a GTM property: a name GTM answers for, from its datacenters.
|
|
type Property struct {
|
|
Name string `json:"name"`
|
|
Type string `json:"type,omitempty"` // failover, weighted-round-robin, performance, …
|
|
Targets []Target `json:"targets"`
|
|
At time.Time `json:"at"` // the availability report's time; zero: not read
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
// Target is one datacenter's servers for a property.
|
|
type Target struct {
|
|
Datacenter string `json:"datacenter"` // its nickname
|
|
ID int `json:"id"`
|
|
Enabled bool `json:"enabled"`
|
|
Weight float64 `json:"weight"`
|
|
Servers []Server `json:"servers"`
|
|
}
|
|
|
|
// Server is one address GTM hands out, as its liveness tests see it.
|
|
type Server struct {
|
|
IP string `json:"ip"`
|
|
Alive bool `json:"alive"`
|
|
HandedOut bool `json:"handed_out"`
|
|
Score float64 `json:"score"`
|
|
Known bool `json:"known"` // the report had it; false: configured but not reported yet
|
|
}
|
|
|
|
// Live says whether the target has a live server.
|
|
func (t Target) Live() bool {
|
|
return slices.ContainsFunc(t.Servers, func(s Server) bool { return s.Alive })
|
|
}
|
|
|
|
// Watcher polls Akamai and keeps the latest reading.
|
|
type Watcher struct {
|
|
cfg Config
|
|
client *http.Client
|
|
mu sync.Mutex
|
|
snap Snapshot
|
|
}
|
|
|
|
// New checks the config; the .edgerc is read on every poll.
|
|
func New(cfg Config) (*Watcher, error) {
|
|
cfg.Defaults()
|
|
if cfg.Edgerc == "" {
|
|
return nil, errors.New("akamai: edgerc_file is required")
|
|
}
|
|
return &Watcher{cfg: cfg, client: &http.Client{Timeout: 30 * time.Second}}, nil
|
|
}
|
|
|
|
func (w *Watcher) Config() Config { return w.cfg }
|
|
|
|
func (w *Watcher) Snapshot() Snapshot {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
return w.snap
|
|
}
|
|
|
|
// Set puts a reading in place (another copy of obs-beacond took it).
|
|
func (w *Watcher) Set(s Snapshot) {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
w.snap = s
|
|
}
|
|
|
|
// Poll reads everything once.
|
|
func (w *Watcher) Poll(ctx context.Context, now time.Time) {
|
|
s, err := w.read(ctx, now)
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
if err != nil {
|
|
if w.snap.Error == "" {
|
|
w.snap.ErrorSince = now
|
|
}
|
|
w.snap.Error = err.Error()
|
|
return // the last good reading stays
|
|
}
|
|
w.snap = s
|
|
}
|
|
|
|
// errAbsent is a product the client can't read: 403 or 404.
|
|
var errAbsent = errors.New("not in this account, or not granted")
|
|
|
|
// errNotAcceptable is a media type the API doesn't serve (406, 415).
|
|
var errNotAcceptable = errors.New("media type not accepted")
|
|
|
|
func (w *Watcher) get(ctx context.Context, c Creds, path string, v any) error {
|
|
return w.getAccept(ctx, c, path, "application/json", v)
|
|
}
|
|
|
|
func (w *Watcher) getAccept(ctx context.Context, c Creds, path, accept string, v any) error {
|
|
return w.do(ctx, c, http.MethodGet, path, accept, nil, v)
|
|
}
|
|
|
|
// do sends one request. Everything is a GET but the traffic report's
|
|
// query, which the Reporting API only takes as a POST; it changes
|
|
// nothing, and no other POST can be sent.
|
|
func (w *Watcher) do(ctx context.Context, c Creds, method, path, accept string, body []byte, v any) error {
|
|
if method != http.MethodGet && !(method == http.MethodPost && path == trafficPath) {
|
|
return fmt.Errorf("akamai: %s %s refused: only reads are sent", method, path)
|
|
}
|
|
base := "https://" + c.Host
|
|
switch {
|
|
case w.cfg.API != "":
|
|
base = w.cfg.API
|
|
case c.Plain:
|
|
base = "http://" + c.Host
|
|
}
|
|
var rd io.Reader
|
|
if body != nil {
|
|
rd = bytes.NewReader(body)
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, method, base+path, rd)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
req.Header.Set("Accept", accept)
|
|
if body != nil {
|
|
req.Header.Set("Content-Type", "application/json")
|
|
}
|
|
// Signed as an https request to the client's host, wherever it goes.
|
|
u := *req.URL
|
|
u.Scheme, u.Host = "https", c.Host
|
|
sreq := &http.Request{Method: req.Method, URL: &u, Header: http.Header{}}
|
|
c.Sign(sreq, body, time.Now())
|
|
req.Header.Set("Authorization", sreq.Header.Get("Authorization"))
|
|
resp, err := w.client.Do(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
switch {
|
|
case resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusNotFound:
|
|
return errAbsent
|
|
case resp.StatusCode == http.StatusNotAcceptable || resp.StatusCode == http.StatusUnsupportedMediaType:
|
|
return errNotAcceptable
|
|
case resp.StatusCode != http.StatusOK:
|
|
var p struct {
|
|
Title string `json:"title"`
|
|
Detail string `json:"detail"`
|
|
}
|
|
b, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
|
_ = json.Unmarshal(b, &p)
|
|
msg := strings.TrimSpace(p.Title + ": " + p.Detail)
|
|
if p.Title == "" {
|
|
msg = resp.Status
|
|
}
|
|
return fmt.Errorf("%s: %s", path, strings.Trim(msg, ": "))
|
|
}
|
|
return json.NewDecoder(io.LimitReader(resp.Body, 32<<20)).Decode(v)
|
|
}
|
|
|
|
// GTM's answers: the fields we use (field names from Akamai's GTM Config
|
|
// and GTM Reporting API references).
|
|
type gtmDomains struct {
|
|
Items []struct {
|
|
Name string `json:"name"`
|
|
} `json:"items"`
|
|
}
|
|
|
|
type gtmDomain struct {
|
|
Name string `json:"name"`
|
|
Type string `json:"type"`
|
|
Datacenters []struct {
|
|
DatacenterID int `json:"datacenterId"`
|
|
Nickname string `json:"nickname"`
|
|
} `json:"datacenters"`
|
|
Properties []struct {
|
|
Name string `json:"name"`
|
|
Type string `json:"type"`
|
|
TrafficTargets []struct {
|
|
DatacenterID int `json:"datacenterId"`
|
|
Enabled bool `json:"enabled"`
|
|
Weight float64 `json:"weight"`
|
|
Servers []string `json:"servers"`
|
|
} `json:"trafficTargets"`
|
|
} `json:"properties"`
|
|
}
|
|
|
|
type gtmStatus struct {
|
|
PropagationStatus string `json:"propagationStatus"`
|
|
PropagationStatusDate time.Time `json:"propagationStatusDate"`
|
|
Message string `json:"message"`
|
|
}
|
|
|
|
type ipAvailability struct {
|
|
DataRows []struct {
|
|
Timestamp time.Time `json:"timestamp"`
|
|
Datacenters []struct {
|
|
DatacenterID int `json:"datacenterId"`
|
|
Nickname string `json:"nickname"`
|
|
IPs []struct {
|
|
IP string `json:"ip"`
|
|
HandedOut bool `json:"handedOut"`
|
|
Score float64 `json:"score"`
|
|
Alive bool `json:"alive"`
|
|
} `json:"IPs"`
|
|
} `json:"datacenters"`
|
|
} `json:"dataRows"`
|
|
}
|
|
|
|
// reportsAtOnce is how many availability reports are read at a time.
|
|
const reportsAtOnce = 4
|
|
|
|
func (w *Watcher) read(ctx context.Context, now time.Time) (Snapshot, error) {
|
|
c, err := ReadEdgerc(w.cfg.Edgerc, w.cfg.Section)
|
|
if err != nil {
|
|
return Snapshot{}, err
|
|
}
|
|
s := Snapshot{Checked: now, Domains: []Domain{}}
|
|
names := w.cfg.Domains
|
|
if len(names) == 0 {
|
|
var ds gtmDomains
|
|
switch err := w.get(ctx, c, "/config-gtm/v1/domains", &ds); {
|
|
case errors.Is(err, errAbsent):
|
|
s.Absent = append(s.Absent, "gtm")
|
|
case err != nil:
|
|
return Snapshot{}, err
|
|
}
|
|
for _, d := range ds.Items {
|
|
names = append(names, d.Name)
|
|
}
|
|
}
|
|
slices.Sort(names)
|
|
for _, n := range names {
|
|
s.Domains = append(s.Domains, w.readDomain(ctx, c, n))
|
|
}
|
|
prev := w.Snapshot()
|
|
switch t, start, end, err := w.readTraffic(ctx, c); {
|
|
case errors.Is(err, errAbsent):
|
|
s.Absent = append(s.Absent, "traffic")
|
|
case err != nil: // the last numbers stay, marked
|
|
s.Traffic, s.TrafficStart, s.TrafficEnd, s.TrafficError = prev.Traffic, prev.TrafficStart, prev.TrafficEnd, err.Error()
|
|
default:
|
|
s.Traffic, s.TrafficStart, s.TrafficEnd = t, start, end
|
|
}
|
|
s.Status = w.status(ctx, c)
|
|
if !prev.CPSAt.IsZero() && now.Sub(prev.CPSAt) < slowEvery {
|
|
s.Enrollments, s.CPSAt, s.CPSError = prev.Enrollments, prev.CPSAt, prev.CPSError
|
|
if slices.Contains(prev.Absent, "cps") {
|
|
s.Absent = append(s.Absent, "cps")
|
|
}
|
|
} else {
|
|
switch es, err := w.readCPS(ctx, c); {
|
|
case errors.Is(err, errAbsent):
|
|
s.Absent = append(s.Absent, "cps")
|
|
s.CPSAt = now
|
|
case err != nil: // GTM's reading still counts; the last enrollments stay
|
|
s.Enrollments, s.CPSError = prev.Enrollments, err.Error()
|
|
default:
|
|
s.Enrollments, s.CPSAt = es, now
|
|
}
|
|
}
|
|
if !prev.PAPIAt.IsZero() && now.Sub(prev.PAPIAt) < slowEvery {
|
|
s.PMProperties, s.PAPIAt, s.PAPIError = prev.PMProperties, prev.PAPIAt, prev.PAPIError
|
|
if slices.Contains(prev.Absent, "papi") {
|
|
s.Absent = append(s.Absent, "papi")
|
|
}
|
|
} else {
|
|
switch ps, err := w.readPAPI(ctx, c, now, prev.PMProperties); {
|
|
case errors.Is(err, errAbsent):
|
|
s.Absent = append(s.Absent, "papi")
|
|
s.PAPIAt = now
|
|
case err != nil:
|
|
s.PMProperties, s.PAPIError = prev.PMProperties, err.Error()
|
|
default:
|
|
s.PMProperties, s.PAPIAt = ps, now
|
|
}
|
|
}
|
|
if !prev.DNSAt.IsZero() && now.Sub(prev.DNSAt) < slowEvery {
|
|
s.DNSZones, s.DNSAt, s.DNSError = prev.DNSZones, prev.DNSAt, prev.DNSError
|
|
if slices.Contains(prev.Absent, "dns") {
|
|
s.Absent = append(s.Absent, "dns")
|
|
}
|
|
} else {
|
|
switch zs, err := w.readDNS(ctx, c, prev.DNSZones); {
|
|
case errors.Is(err, errAbsent):
|
|
s.Absent = append(s.Absent, "dns")
|
|
s.DNSAt = now
|
|
case err != nil:
|
|
s.DNSZones, s.DNSError = prev.DNSZones, err.Error()
|
|
default:
|
|
s.DNSZones, s.DNSAt = zs, now
|
|
}
|
|
}
|
|
return s, nil
|
|
}
|
|
|
|
// slowEvery is how often CPS, Property Manager and Edge DNS are read: they change
|
|
// slowly, and their APIs' rate limits are per account (docs/akamai.md,
|
|
// Scale).
|
|
const slowEvery = 15 * time.Minute
|
|
|
|
func (w *Watcher) readDomain(ctx context.Context, c Creds, name string) Domain {
|
|
d := Domain{Name: name, Properties: []Property{}}
|
|
esc := url.PathEscape(name)
|
|
var g gtmDomain
|
|
if err := w.get(ctx, c, "/config-gtm/v1/domains/"+esc, &g); err != nil {
|
|
d.Error = err.Error()
|
|
return d
|
|
}
|
|
d.Type = g.Type
|
|
var st gtmStatus
|
|
if err := w.get(ctx, c, "/config-gtm/v1/domains/"+esc+"/status/current", &st); err == nil {
|
|
d.Propagation, d.Since, d.Message = st.PropagationStatus, st.PropagationStatusDate, st.Message
|
|
}
|
|
dcName := map[int]string{}
|
|
for _, dc := range g.Datacenters {
|
|
dcName[dc.DatacenterID] = dc.Nickname
|
|
}
|
|
d.Properties = make([]Property, len(g.Properties))
|
|
var wg sync.WaitGroup
|
|
sem := make(chan struct{}, reportsAtOnce)
|
|
for i, gp := range g.Properties {
|
|
p := Property{Name: gp.Name, Type: gp.Type, Targets: []Target{}}
|
|
for _, t := range gp.TrafficTargets {
|
|
tg := Target{Datacenter: dcName[t.DatacenterID], ID: t.DatacenterID, Enabled: t.Enabled, Weight: t.Weight, Servers: []Server{}}
|
|
if tg.Datacenter == "" {
|
|
tg.Datacenter = fmt.Sprintf("datacenter %d", t.DatacenterID)
|
|
}
|
|
for _, ip := range t.Servers {
|
|
tg.Servers = append(tg.Servers, Server{IP: ip})
|
|
}
|
|
p.Targets = append(p.Targets, tg)
|
|
}
|
|
d.Properties[i] = p
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
sem <- struct{}{}
|
|
defer func() { <-sem }()
|
|
w.availability(ctx, c, name, &d.Properties[i])
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
for i := range d.Properties {
|
|
slices.SortFunc(d.Properties[i].Targets, func(a, b Target) int { return strings.Compare(a.Datacenter, b.Datacenter) })
|
|
}
|
|
slices.SortFunc(d.Properties, func(a, b Property) int { return strings.Compare(a.Name, b.Name) })
|
|
return d
|
|
}
|
|
|
|
// availability fills in each server's liveness from the most recent
|
|
// IP availability report.
|
|
func (w *Watcher) availability(ctx context.Context, c Creds, domain string, p *Property) {
|
|
var r ipAvailability
|
|
path := "/gtm-api/v1/reports/ip-availability/domains/" + url.PathEscape(domain) + "/properties/" + url.PathEscape(p.Name) + "?mostRecent=true"
|
|
if err := w.get(ctx, c, path, &r); err != nil {
|
|
p.Error = err.Error()
|
|
return
|
|
}
|
|
if len(r.DataRows) == 0 {
|
|
return
|
|
}
|
|
row := r.DataRows[len(r.DataRows)-1]
|
|
p.At = row.Timestamp
|
|
for _, dc := range row.Datacenters {
|
|
for ti := range p.Targets {
|
|
t := &p.Targets[ti]
|
|
if t.ID != dc.DatacenterID {
|
|
continue
|
|
}
|
|
for _, ip := range dc.IPs {
|
|
i := slices.IndexFunc(t.Servers, func(s Server) bool { return s.IP == ip.IP })
|
|
if i < 0 {
|
|
t.Servers = append(t.Servers, Server{IP: ip.IP})
|
|
i = len(t.Servers) - 1
|
|
}
|
|
t.Servers[i].Alive, t.Servers[i].HandedOut, t.Servers[i].Score, t.Servers[i].Known = ip.Alive, ip.HandedOut, ip.Score, true
|
|
}
|
|
}
|
|
}
|
|
}
|