Files
cairnobs/scripts/fleet/akamai.go
T
jcoffey-dev 95965ed5e8
ci / postgres (pull_request) Successful in 3m14s
ci / check (pull_request) Successful in 8m22s
Demo: hosts named for their site and role, collectors on their own
Things that gave the simulator away (#40):

- hosts were fleet-0000 to fleet-0119. Now they're named for their site
  and what they do: phx1-gw01, phx1-lb01, phx1-app05, phx1-db03,
  phx1-cache02, iad1-lb02, lon1-kvm01 (collectors.go);
- fleet-0041 was a Redis cache, a KVM host, the IBM i collector and the
  HMC reader at once, so IBM i alerts read as if PROD2 ran on it. The
  agents that read storage, IBM i and virtualization are small hosts of
  their own now: phx1-stor01 (the Pure arrays), phx1-ibmi01 (the
  partitions and the HMC), phx1-virt01 (vCenter, Proxmox, Nutanix).
  fleet config writes their agents;
- libvirt runs only on app servers, not on a load balancer or a cache;
- RHEL hosts' units are sshd and chronyd, as RHEL names them;
- Akamai's CP codes no longer have round counts.

Varied disks and jittered readings came with #39 (load.go).
2026-11-15 23:20:21 -07:00

239 lines
9.6 KiB
Go

package main
import (
"context"
"fmt"
"log"
"net"
"net/http"
"os"
"path/filepath"
"time"
"github.com/cairnobs/cairnobs/internal/akamai"
)
// The fleet's fake Akamai (docs/akamai.md, step 5): an enterprise
// account on loopback, every signature checked. GTM's datacenters are
// the first sites and its servers their load balancers' public
// addresses (netmap.go, positions 1 and 2), so a failing one is a fleet
// host. Problems on purpose, the ones marked * coming and going
// (scenario.go):
//
// GTM * site 2's first load balancer failing for api.shop;
// * the legacy property with no live datacenter (story 3)
// CPS 180 enrollments: a third-party renewal waiting on a CSR,
// ending in 15 days
// Property 320 properties: a failed production activation;
// Manager * one rolling out for longer than it should
// Edge DNS 40 zones (one of 5,000 records); one changes an MX 20
// minutes after the start
// Traffic 320 CP codes; * the legacy site's at 30% 5xx while its
// datacenter is down (story 3)
const akamaiPort = 58410
func serveAkamai(ctx context.Context, dir string) {
if err := os.MkdirAll(dir, 0o700); err != nil {
log.Fatal(err)
}
ln, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", akamaiPort))
if err != nil {
log.Fatal(err)
}
c := akamai.Creds{Host: ln.Addr().String(), ClientToken: "akab-fleet-client", ClientSecret: "ZmxlZXQtc2VjcmV0", AccessToken: "akab-fleet-access"}
edgerc := filepath.Join(dir, ".edgerc")
body := fmt.Sprintf("[default]\nhost = http://%s\nclient_token = %s\nclient_secret = %s\naccess_token = %s\n", c.Host, c.ClientToken, c.ClientSecret, c.AccessToken)
if err := os.WriteFile(edgerc, []byte(body), 0o600); err != nil {
log.Fatal(err)
}
f := &akamai.Fake{Creds: c}
start := time.Now()
n := fleetAkamai(f)
go func() {
t := time.NewTicker(30 * time.Second)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case now := <-t.C:
akamaiStory(f, now)
}
}
}()
srv := &http.Server{Handler: f, ReadHeaderTimeout: 5 * time.Second}
go func() { _ = srv.Serve(ln) }()
go func() { <-ctx.Done(); _ = srv.Close() }()
time.AfterFunc(20*time.Minute, func() {
f.SetZone("zone07.example", akamai.FakeZone{Records: fleetZone(7, "10 mx.relay.example.net.")})
log.Print("fleet: Akamai Edge DNS: zone07.example's MX changed")
})
yaml := "akamai:\n edgerc_file: " + edgerc + "\n"
p := filepath.Join(dir, "akamai.yaml")
if err := os.WriteFile(p, []byte(yaml), 0o600); err != nil {
log.Fatal(err)
}
log.Printf("fleet: Akamai at http://%s (%s, made in %s); %s has the akamai: section for obs-beacond.yaml", c.Host, n, time.Since(start).Round(time.Millisecond), p)
}
// fleetAkamai fills the fake and says what it has.
func fleetAkamai(f *akamai.Fake) string {
day := 24 * time.Hour
now := time.Now()
// GTM: datacenters are sites 0-5; each property's targets are three
// of them, servers their load balancers' eth0 addresses.
sites := min(fleetSites, 6)
dcs := map[int]string{}
cities := []string{"Ashburn", "Frankfurt", "Singapore", "Dallas", "London", "Tokyo"}
for s := range sites {
dcs[3100+s] = cities[s]
}
lb := func(site, pos int) string { return fmt.Sprintf("203.0.%d.%d", site%250, 10+pos) }
props := 0
for d, name := range []string{"shop", "api", "media", "corp", "eu", "apac", "status", "auth"} {
var ps []akamai.FakeProperty
for p := range 6 + d%3*3 {
targets := map[int]map[string]bool{}
for k := range min(3, sites) {
s := (p + d + k) % sites
targets[3100+s] = map[string]bool{lb(s, 1): true, lb(s, 2): true}
}
ps = append(ps, akamai.FakeProperty{Name: fmt.Sprintf("p%02d", p), Type: "performance", Targets: targets})
}
switch name {
case "shop":
ps[0].Name = "api"
case "corp":
ps = append(ps, akamai.FakeProperty{Name: "legacy", Type: "failover", Targets: map[int]map[string]bool{3100: {legacyServer: true}}})
}
props += len(ps)
f.SetDomain(name+".example.akadns.net", akamai.FakeDomain{Type: "full", Datacenters: dcs, Properties: ps})
}
// CPS: ECDSA keys (fast to make); every tenth has an RSA one too.
for i := range 180 {
cn := fmt.Sprintf("app%03d.example.com", i)
left := time.Duration(25+(i*37)%300) * day
if i == 47 {
left = 15 * day
}
e := akamai.FakeEnrollment{CN: cn, SANs: []string{cn, "www." + cn}, Validation: "dv", Contract: "1-FLEET", Waiting: "-"}
issuer := "Let's Encrypt E7"
if i%6 == 0 {
e.Validation, issuer = "ov", "DigiCert Global G2 TLS RSA SHA256 2020 CA1"
}
if i == 47 {
e.Validation, e.Waiting, issuer = "third-party", "third-party-csr", "Sectigo RSA Domain Validation Secure Server CA"
}
e.Deployed = []string{akamai.FakePEM(issuer, e.SANs, true, now.Add(left))}
if i%10 == 0 {
e.Deployed = append(e.Deployed, akamai.FakePEM(issuer, e.SANs, false, now.Add(left)))
}
f.SetEnrollment(200000+i, e)
}
// Property Manager: six groups over two contracts.
groups := []string{"Shop", "Media", "Corporate", "APIs", "Europe", "Asia"}
for i := range 320 {
v := 3 + (i*7)%60
p := akamai.FakePMProperty{Name: fmt.Sprintf("site%03d.example.com", i), Group: groups[i%len(groups)], Contract: []string{"1-FLEET", "2-FLEET"}[i%2],
Latest: v, Staging: v, Production: v, Activations: []akamai.FakeActivation{
{Version: v, Network: "STAGING", Status: "ACTIVE", Submitted: now.Add(-time.Duration(30+i%500) * time.Hour)},
{Version: v, Network: "PRODUCTION", Status: "ACTIVE", Submitted: now.Add(-time.Duration(29+i%500) * time.Hour)}}}
// v+1: on staging first, then production.
staged := akamai.FakeActivation{Version: v + 1, Network: "STAGING", Status: "ACTIVE", Submitted: now.Add(-5 * time.Hour)}
switch i {
case 4:
p.Latest, p.Staging = v+1, v+1
p.Activations = append(p.Activations, staged,
akamai.FakeActivation{Version: v + 1, Network: "PRODUCTION", Status: "FAILED", Submitted: now.Add(-3 * time.Hour), Note: "Origin moved to the new load balancers"})
}
f.SetPMProperty(fmt.Sprintf("prp_9%05d", i), p)
if i == slowActivation {
slowBase = p
}
}
// Edge DNS.
records := 0
for z := range 40 {
recs := fleetZone(z, "10 mx1.example.com.")
if z == 0 {
for h := range 5000 {
recs = append(recs, akamai.RecordSet{Name: fmt.Sprintf("host%04d.zone00.example", h), Type: "A", TTL: 300, RData: []string{fmt.Sprintf("10.%d.%d.%d", 100+h/62500, h/250%250, h%250+1)}})
}
}
records += len(recs)
f.SetZone(fmt.Sprintf("zone%02d.example", z), akamai.FakeZone{Records: recs})
}
// Traffic: a long tail, about one request in 2,000 failing, no two
// CP codes alike.
for i := range 320 {
hits, errs := cpTraffic(int64(400000 + i))
f.SetTraffic(int64(400000+i), hits, errs)
}
akamaiStory(f, now)
return fmt.Sprintf("%d GTM properties over %d datacenters, 180 CPS enrollments, 320 properties, 40 DNS zones with %d record sets, 320 CP codes", props, sites, records)
}
const (
legacyServer = "198.51.100.50" // the legacy property's one datacenter's server
legacyCPCode = 400019 // the legacy site's traffic
slowActivation = 15 // the property whose production activation drags
)
// cpTraffic is a CP code's requests and errors in a reading: the long
// tail's share for its place in it, give or take 15%, and one in 1,600 to
// 2,600 failing.
func cpTraffic(cp int64) (hits, errs int64) {
k := fmt.Sprint(cp)
hits = int64(float64(2_000_000/(cp-400000+1)) * (0.85 + 0.3*seed(k, "hits")))
return hits, hits / int64(1600+1000*seed(k, "errors"))
}
// slowBase is the slow property as fleetAkamai made it, before its new
// version.
var slowBase akamai.FakePMProperty
// akamaiStory puts the problems that come and go in their state now
// (scenario.go): story 3, the legacy datacenter; a load balancer that
// fails GTM's liveness test now and then; an activation that drags.
func akamaiStory(f *akamai.Fake, now time.Time) {
legacyDown := spellLegacyDC.active(now)
f.SetAlive("corp.example.akadns.net", "legacy", 3100, legacyServer, !legacyDown)
hits, errs := cpTraffic(legacyCPCode)
if legacyDown {
errs = hits * 30 / 100
}
f.SetTraffic(legacyCPCode, hits, errs)
if sites := min(fleetSites, 6); sites > 2 {
f.SetAlive("shop.example.akadns.net", "api", 3100+2, fmt.Sprintf("203.0.%d.%d", 2, 11), !spellLBFlaps.active(now))
}
// Its new version goes to production at the start of each spell and
// is active an hour and twenty minutes later.
p := slowBase
p.Activations = append([]akamai.FakeActivation{}, p.Activations...)
v := p.Latest + 1
b := spellActivation.began(now)
p.Latest, p.Staging = v, v
p.Activations = append(p.Activations, akamai.FakeActivation{Version: v, Network: "STAGING", Status: "ACTIVE", Submitted: b.Add(-2 * time.Hour)})
if spellActivation.active(now) {
p.Activations = append(p.Activations, akamai.FakeActivation{Version: v, Network: "PRODUCTION", Status: "PENDING", Submitted: b})
} else {
p.Production = v
p.Activations = append(p.Activations, akamai.FakeActivation{Version: v, Network: "PRODUCTION", Status: "ACTIVE", Submitted: b})
}
f.SetPMProperty(fmt.Sprintf("prp_9%05d", slowActivation), p)
}
// fleetZone is zone z's ordinary records, with the given MX.
func fleetZone(z int, mx string) []akamai.RecordSet {
name := fmt.Sprintf("zone%02d.example", z)
recs := []akamai.RecordSet{
{Name: name, Type: "MX", TTL: 3600, RData: []string{mx}},
{Name: name, Type: "TXT", TTL: 3600, RData: []string{`"v=spf1 mx -all"`}},
{Name: "_dmarc." + name, Type: "TXT", TTL: 3600, RData: []string{`"v=DMARC1; p=reject"`}},
}
for h := range 20 + z*5 {
recs = append(recs, akamai.RecordSet{Name: fmt.Sprintf("www%d.%s", h, name), Type: "CNAME", TTL: 300, RData: []string{fmt.Sprintf("www%d.%s.edgekey.net.", h, name)}})
}
return recs
}