Things that gave the simulator away (#40): - hosts were fleet-0000 to fleet-0119. Now they're named for their site and what they do: phx1-gw01, phx1-lb01, phx1-app05, phx1-db03, phx1-cache02, iad1-lb02, lon1-kvm01 (collectors.go); - fleet-0041 was a Redis cache, a KVM host, the IBM i collector and the HMC reader at once, so IBM i alerts read as if PROD2 ran on it. The agents that read storage, IBM i and virtualization are small hosts of their own now: phx1-stor01 (the Pure arrays), phx1-ibmi01 (the partitions and the HMC), phx1-virt01 (vCenter, Proxmox, Nutanix). fleet config writes their agents; - libvirt runs only on app servers, not on a load balancer or a cache; - RHEL hosts' units are sshd and chronyd, as RHEL names them; - Akamai's CP codes no longer have round counts. Varied disks and jittered readings came with #39 (load.go).
239 lines
9.6 KiB
Go
239 lines
9.6 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log"
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"time"
|
|
|
|
"github.com/cairnobs/cairnobs/internal/akamai"
|
|
)
|
|
|
|
// The fleet's fake Akamai (docs/akamai.md, step 5): an enterprise
|
|
// account on loopback, every signature checked. GTM's datacenters are
|
|
// the first sites and its servers their load balancers' public
|
|
// addresses (netmap.go, positions 1 and 2), so a failing one is a fleet
|
|
// host. Problems on purpose, the ones marked * coming and going
|
|
// (scenario.go):
|
|
//
|
|
// GTM * site 2's first load balancer failing for api.shop;
|
|
// * the legacy property with no live datacenter (story 3)
|
|
// CPS 180 enrollments: a third-party renewal waiting on a CSR,
|
|
// ending in 15 days
|
|
// Property 320 properties: a failed production activation;
|
|
// Manager * one rolling out for longer than it should
|
|
// Edge DNS 40 zones (one of 5,000 records); one changes an MX 20
|
|
// minutes after the start
|
|
// Traffic 320 CP codes; * the legacy site's at 30% 5xx while its
|
|
// datacenter is down (story 3)
|
|
const akamaiPort = 58410
|
|
|
|
func serveAkamai(ctx context.Context, dir string) {
|
|
if err := os.MkdirAll(dir, 0o700); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
ln, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", akamaiPort))
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
c := akamai.Creds{Host: ln.Addr().String(), ClientToken: "akab-fleet-client", ClientSecret: "ZmxlZXQtc2VjcmV0", AccessToken: "akab-fleet-access"}
|
|
edgerc := filepath.Join(dir, ".edgerc")
|
|
body := fmt.Sprintf("[default]\nhost = http://%s\nclient_token = %s\nclient_secret = %s\naccess_token = %s\n", c.Host, c.ClientToken, c.ClientSecret, c.AccessToken)
|
|
if err := os.WriteFile(edgerc, []byte(body), 0o600); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
f := &akamai.Fake{Creds: c}
|
|
start := time.Now()
|
|
n := fleetAkamai(f)
|
|
go func() {
|
|
t := time.NewTicker(30 * time.Second)
|
|
defer t.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case now := <-t.C:
|
|
akamaiStory(f, now)
|
|
}
|
|
}
|
|
}()
|
|
srv := &http.Server{Handler: f, ReadHeaderTimeout: 5 * time.Second}
|
|
go func() { _ = srv.Serve(ln) }()
|
|
go func() { <-ctx.Done(); _ = srv.Close() }()
|
|
time.AfterFunc(20*time.Minute, func() {
|
|
f.SetZone("zone07.example", akamai.FakeZone{Records: fleetZone(7, "10 mx.relay.example.net.")})
|
|
log.Print("fleet: Akamai Edge DNS: zone07.example's MX changed")
|
|
})
|
|
yaml := "akamai:\n edgerc_file: " + edgerc + "\n"
|
|
p := filepath.Join(dir, "akamai.yaml")
|
|
if err := os.WriteFile(p, []byte(yaml), 0o600); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
log.Printf("fleet: Akamai at http://%s (%s, made in %s); %s has the akamai: section for obs-beacond.yaml", c.Host, n, time.Since(start).Round(time.Millisecond), p)
|
|
}
|
|
|
|
// fleetAkamai fills the fake and says what it has.
|
|
func fleetAkamai(f *akamai.Fake) string {
|
|
day := 24 * time.Hour
|
|
now := time.Now()
|
|
// GTM: datacenters are sites 0-5; each property's targets are three
|
|
// of them, servers their load balancers' eth0 addresses.
|
|
sites := min(fleetSites, 6)
|
|
dcs := map[int]string{}
|
|
cities := []string{"Ashburn", "Frankfurt", "Singapore", "Dallas", "London", "Tokyo"}
|
|
for s := range sites {
|
|
dcs[3100+s] = cities[s]
|
|
}
|
|
lb := func(site, pos int) string { return fmt.Sprintf("203.0.%d.%d", site%250, 10+pos) }
|
|
props := 0
|
|
for d, name := range []string{"shop", "api", "media", "corp", "eu", "apac", "status", "auth"} {
|
|
var ps []akamai.FakeProperty
|
|
for p := range 6 + d%3*3 {
|
|
targets := map[int]map[string]bool{}
|
|
for k := range min(3, sites) {
|
|
s := (p + d + k) % sites
|
|
targets[3100+s] = map[string]bool{lb(s, 1): true, lb(s, 2): true}
|
|
}
|
|
ps = append(ps, akamai.FakeProperty{Name: fmt.Sprintf("p%02d", p), Type: "performance", Targets: targets})
|
|
}
|
|
switch name {
|
|
case "shop":
|
|
ps[0].Name = "api"
|
|
case "corp":
|
|
ps = append(ps, akamai.FakeProperty{Name: "legacy", Type: "failover", Targets: map[int]map[string]bool{3100: {legacyServer: true}}})
|
|
}
|
|
props += len(ps)
|
|
f.SetDomain(name+".example.akadns.net", akamai.FakeDomain{Type: "full", Datacenters: dcs, Properties: ps})
|
|
}
|
|
// CPS: ECDSA keys (fast to make); every tenth has an RSA one too.
|
|
for i := range 180 {
|
|
cn := fmt.Sprintf("app%03d.example.com", i)
|
|
left := time.Duration(25+(i*37)%300) * day
|
|
if i == 47 {
|
|
left = 15 * day
|
|
}
|
|
e := akamai.FakeEnrollment{CN: cn, SANs: []string{cn, "www." + cn}, Validation: "dv", Contract: "1-FLEET", Waiting: "-"}
|
|
issuer := "Let's Encrypt E7"
|
|
if i%6 == 0 {
|
|
e.Validation, issuer = "ov", "DigiCert Global G2 TLS RSA SHA256 2020 CA1"
|
|
}
|
|
if i == 47 {
|
|
e.Validation, e.Waiting, issuer = "third-party", "third-party-csr", "Sectigo RSA Domain Validation Secure Server CA"
|
|
}
|
|
e.Deployed = []string{akamai.FakePEM(issuer, e.SANs, true, now.Add(left))}
|
|
if i%10 == 0 {
|
|
e.Deployed = append(e.Deployed, akamai.FakePEM(issuer, e.SANs, false, now.Add(left)))
|
|
}
|
|
f.SetEnrollment(200000+i, e)
|
|
}
|
|
// Property Manager: six groups over two contracts.
|
|
groups := []string{"Shop", "Media", "Corporate", "APIs", "Europe", "Asia"}
|
|
for i := range 320 {
|
|
v := 3 + (i*7)%60
|
|
p := akamai.FakePMProperty{Name: fmt.Sprintf("site%03d.example.com", i), Group: groups[i%len(groups)], Contract: []string{"1-FLEET", "2-FLEET"}[i%2],
|
|
Latest: v, Staging: v, Production: v, Activations: []akamai.FakeActivation{
|
|
{Version: v, Network: "STAGING", Status: "ACTIVE", Submitted: now.Add(-time.Duration(30+i%500) * time.Hour)},
|
|
{Version: v, Network: "PRODUCTION", Status: "ACTIVE", Submitted: now.Add(-time.Duration(29+i%500) * time.Hour)}}}
|
|
// v+1: on staging first, then production.
|
|
staged := akamai.FakeActivation{Version: v + 1, Network: "STAGING", Status: "ACTIVE", Submitted: now.Add(-5 * time.Hour)}
|
|
switch i {
|
|
case 4:
|
|
p.Latest, p.Staging = v+1, v+1
|
|
p.Activations = append(p.Activations, staged,
|
|
akamai.FakeActivation{Version: v + 1, Network: "PRODUCTION", Status: "FAILED", Submitted: now.Add(-3 * time.Hour), Note: "Origin moved to the new load balancers"})
|
|
}
|
|
f.SetPMProperty(fmt.Sprintf("prp_9%05d", i), p)
|
|
if i == slowActivation {
|
|
slowBase = p
|
|
}
|
|
}
|
|
// Edge DNS.
|
|
records := 0
|
|
for z := range 40 {
|
|
recs := fleetZone(z, "10 mx1.example.com.")
|
|
if z == 0 {
|
|
for h := range 5000 {
|
|
recs = append(recs, akamai.RecordSet{Name: fmt.Sprintf("host%04d.zone00.example", h), Type: "A", TTL: 300, RData: []string{fmt.Sprintf("10.%d.%d.%d", 100+h/62500, h/250%250, h%250+1)}})
|
|
}
|
|
}
|
|
records += len(recs)
|
|
f.SetZone(fmt.Sprintf("zone%02d.example", z), akamai.FakeZone{Records: recs})
|
|
}
|
|
// Traffic: a long tail, about one request in 2,000 failing, no two
|
|
// CP codes alike.
|
|
for i := range 320 {
|
|
hits, errs := cpTraffic(int64(400000 + i))
|
|
f.SetTraffic(int64(400000+i), hits, errs)
|
|
}
|
|
akamaiStory(f, now)
|
|
return fmt.Sprintf("%d GTM properties over %d datacenters, 180 CPS enrollments, 320 properties, 40 DNS zones with %d record sets, 320 CP codes", props, sites, records)
|
|
}
|
|
|
|
const (
|
|
legacyServer = "198.51.100.50" // the legacy property's one datacenter's server
|
|
legacyCPCode = 400019 // the legacy site's traffic
|
|
slowActivation = 15 // the property whose production activation drags
|
|
)
|
|
|
|
// cpTraffic is a CP code's requests and errors in a reading: the long
|
|
// tail's share for its place in it, give or take 15%, and one in 1,600 to
|
|
// 2,600 failing.
|
|
func cpTraffic(cp int64) (hits, errs int64) {
|
|
k := fmt.Sprint(cp)
|
|
hits = int64(float64(2_000_000/(cp-400000+1)) * (0.85 + 0.3*seed(k, "hits")))
|
|
return hits, hits / int64(1600+1000*seed(k, "errors"))
|
|
}
|
|
|
|
// slowBase is the slow property as fleetAkamai made it, before its new
|
|
// version.
|
|
var slowBase akamai.FakePMProperty
|
|
|
|
// akamaiStory puts the problems that come and go in their state now
|
|
// (scenario.go): story 3, the legacy datacenter; a load balancer that
|
|
// fails GTM's liveness test now and then; an activation that drags.
|
|
func akamaiStory(f *akamai.Fake, now time.Time) {
|
|
legacyDown := spellLegacyDC.active(now)
|
|
f.SetAlive("corp.example.akadns.net", "legacy", 3100, legacyServer, !legacyDown)
|
|
hits, errs := cpTraffic(legacyCPCode)
|
|
if legacyDown {
|
|
errs = hits * 30 / 100
|
|
}
|
|
f.SetTraffic(legacyCPCode, hits, errs)
|
|
if sites := min(fleetSites, 6); sites > 2 {
|
|
f.SetAlive("shop.example.akadns.net", "api", 3100+2, fmt.Sprintf("203.0.%d.%d", 2, 11), !spellLBFlaps.active(now))
|
|
}
|
|
// Its new version goes to production at the start of each spell and
|
|
// is active an hour and twenty minutes later.
|
|
p := slowBase
|
|
p.Activations = append([]akamai.FakeActivation{}, p.Activations...)
|
|
v := p.Latest + 1
|
|
b := spellActivation.began(now)
|
|
p.Latest, p.Staging = v, v
|
|
p.Activations = append(p.Activations, akamai.FakeActivation{Version: v, Network: "STAGING", Status: "ACTIVE", Submitted: b.Add(-2 * time.Hour)})
|
|
if spellActivation.active(now) {
|
|
p.Activations = append(p.Activations, akamai.FakeActivation{Version: v, Network: "PRODUCTION", Status: "PENDING", Submitted: b})
|
|
} else {
|
|
p.Production = v
|
|
p.Activations = append(p.Activations, akamai.FakeActivation{Version: v, Network: "PRODUCTION", Status: "ACTIVE", Submitted: b})
|
|
}
|
|
f.SetPMProperty(fmt.Sprintf("prp_9%05d", slowActivation), p)
|
|
}
|
|
|
|
// fleetZone is zone z's ordinary records, with the given MX.
|
|
func fleetZone(z int, mx string) []akamai.RecordSet {
|
|
name := fmt.Sprintf("zone%02d.example", z)
|
|
recs := []akamai.RecordSet{
|
|
{Name: name, Type: "MX", TTL: 3600, RData: []string{mx}},
|
|
{Name: name, Type: "TXT", TTL: 3600, RData: []string{`"v=spf1 mx -all"`}},
|
|
{Name: "_dmarc." + name, Type: "TXT", TTL: 3600, RData: []string{`"v=DMARC1; p=reject"`}},
|
|
}
|
|
for h := range 20 + z*5 {
|
|
recs = append(recs, akamai.RecordSet{Name: fmt.Sprintf("www%d.%s", h, name), Type: "CNAME", TTL: 300, RData: []string{fmt.Sprintf("www%d.%s.edgekey.net.", h, name)}})
|
|
}
|
|
return recs
|
|
}
|