Every graph on the demo said "none kept yet" and the reports had nothing to report: the container starts fresh each week. Now fleet run -history <database> -state-key <file> first writes the last 30 days into obs-beacond's database, from the same plan and clock the live demo runs on: - hosts' hours: what a host uses is now a function of its name and the clock (load.go): a working day in its site's time zone, quieter weekends, memory that creeps on some hosts until a weekly restart, and disks at a level of their own that fill slowly until a clean-up (fleet-0023's log volume warns now and then). No more every disk at 33%, part of #40; - the clusters' workloads, the arrays' I/O and space, the IBM i partitions' CPU, the Vaults' leases (on a daily curve now), the mail rounds; - hosts', checks' and services' statuses, for the reports' availability; - an incident each time a past problem came and went (pastProblems in scenario.go, with obs-beacond's keys and titles), notes on the stories saying what the cause was, and the incident numbers carrying on. About 45 seconds for the full demo. state gains PutHostHours, PutMailRounds and LogStatuses, to write many in one transaction. docs/demo.md and docs/development.md describe it. Closes #39.
282 lines
12 KiB
Go
282 lines
12 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"crypto/ecdsa"
|
|
"crypto/elliptic"
|
|
"crypto/rand"
|
|
"crypto/sha256"
|
|
"crypto/tls"
|
|
"crypto/x509"
|
|
"crypto/x509/pkix"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"encoding/pem"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"math"
|
|
"math/big"
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Two Vault clusters obs-beacond reads itself (docs/vault.md: the fleet
|
|
// simulator), served over HTTPS on loopback:
|
|
//
|
|
// - vault-prod: Vault Enterprise, three healthy Raft servers, DR
|
|
// replication streaming to its secondary, a license 300 days out, an
|
|
// audit device, and a PKI mount (pki_int) with 400 services'
|
|
// certificates (one ending within a week, its renewal overdue) and
|
|
// 2,000 short-lived ones.
|
|
// - vault-edge: OpenBao, three Raft servers, one of which fails now and
|
|
// then and leaves no failure tolerance (scenario.go), and an
|
|
// intermediate CA 12 days from its end.
|
|
//
|
|
// They listen on vaultPort and the port after it; -vault's directory
|
|
// keeps their certificate and key, so vault.yaml (written there) stays
|
|
// the same across runs.
|
|
|
|
const vaultPort = 58310
|
|
|
|
type fakeVault struct {
|
|
name, version string
|
|
enterprise bool
|
|
servers []fakeRaft
|
|
ft int
|
|
mount string
|
|
issuer string // PEM
|
|
issuerID string
|
|
certs map[string]string // serial → PEM
|
|
serials []string
|
|
revoked []string
|
|
leases int
|
|
}
|
|
|
|
type fakeRaft struct{ id, status, node string }
|
|
|
|
func serveVaults(ctx context.Context, dir string) {
|
|
if err := os.MkdirAll(dir, 0o700); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
cert, fp, err := vaultTLS(dir)
|
|
if err != nil {
|
|
log.Fatalf("fleet: vault: %v", err)
|
|
}
|
|
tok := filepath.Join(dir, "token")
|
|
if err := os.WriteFile(tok, []byte("fleet-vault-token\n"), 0o600); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
now := time.Now()
|
|
vaults := []*fakeVault{
|
|
newFakeVault("vault-prod", "1.21.4+ent", true, "pki_int", now, 300*24*time.Hour, 400, 2000,
|
|
[]fakeRaft{{"v1", "leader", "alive"}, {"v2", "voter", "alive"}, {"v3", "voter", "alive"}}, 1, vaultLeaseBase["vault-prod"]),
|
|
newFakeVault("vault-edge", "2.4.1", false, "pki_edge", now, 12*24*time.Hour, 40, 0,
|
|
[]fakeRaft{{"e1", "leader", "alive"}, {"e2", "voter", "alive"}, {"e3", "voter", "alive"}}, 1, vaultLeaseBase["vault-edge"]),
|
|
}
|
|
var yaml strings.Builder
|
|
yaml.WriteString("vault:\n")
|
|
for i, v := range vaults {
|
|
ln, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", vaultPort+i))
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
srv := &http.Server{Handler: v, TLSConfig: &tls.Config{Certificates: []tls.Certificate{cert}}, ReadHeaderTimeout: 5 * time.Second}
|
|
go func() { _ = srv.ServeTLS(ln, "", "") }()
|
|
go func() { <-ctx.Done(); _ = srv.Close() }()
|
|
fmt.Fprintf(&yaml, " - {name: %s, url: 'https://%s', token_file: %s, fingerprint: %s}\n", v.name, ln.Addr(), tok, fp)
|
|
log.Printf("fleet: Vault %s (%s) at https://%s: %d certificates on %s", v.name, v.version, ln.Addr(), len(v.serials), v.mount)
|
|
}
|
|
p := filepath.Join(dir, "vault.yaml")
|
|
if err := os.WriteFile(p, []byte(yaml.String()), 0o600); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
log.Printf("fleet: %s has the vault: list for obs-beacond.yaml", p)
|
|
}
|
|
|
|
// vaultTLS is the fake Vaults' certificate, made once and kept in dir.
|
|
func vaultTLS(dir string) (tls.Certificate, string, error) {
|
|
cp, kp := filepath.Join(dir, "tls.crt"), filepath.Join(dir, "tls.key")
|
|
if _, err := os.Stat(cp); errors.Is(err, os.ErrNotExist) {
|
|
key, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
|
tpl := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "fleet vault"}, DNSNames: []string{"localhost"},
|
|
IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)}, NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(10 * 365 * 24 * time.Hour)}
|
|
der, err := x509.CreateCertificate(rand.Reader, tpl, tpl, &key.PublicKey, key)
|
|
if err != nil {
|
|
return tls.Certificate{}, "", err
|
|
}
|
|
kb, _ := x509.MarshalECPrivateKey(key)
|
|
_ = os.WriteFile(cp, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600)
|
|
_ = os.WriteFile(kp, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: kb}), 0o600)
|
|
}
|
|
c, err := tls.LoadX509KeyPair(cp, kp)
|
|
if err != nil {
|
|
return c, "", err
|
|
}
|
|
sum := sha256.Sum256(c.Certificate[0])
|
|
return c, hex.EncodeToString(sum[:]), nil
|
|
}
|
|
|
|
// newFakeVault makes a cluster with an intermediate CA ending caLeft from
|
|
// now, a year's certificates for that many services (most renewed once)
|
|
// and short ones lasting an hour.
|
|
func newFakeVault(name, version string, ent bool, mount string, now time.Time, caLeft time.Duration, services, short int,
|
|
servers []fakeRaft, ft, leases int) *fakeVault {
|
|
v := &fakeVault{name: name, version: version, enterprise: ent, servers: servers, ft: ft, mount: mount, certs: map[string]string{},
|
|
issuerID: "4f6c1e2a-0000-4000-8000-" + fmt.Sprintf("%012d", len(name)), leases: leases}
|
|
caKey, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
|
n := int64(1000)
|
|
sign := func(tpl *x509.Certificate, parent *x509.Certificate, pub any, by *ecdsa.PrivateKey) (*x509.Certificate, string) {
|
|
n++
|
|
tpl.SerialNumber = big.NewInt(n)
|
|
if parent == nil {
|
|
parent = tpl
|
|
}
|
|
der, err := x509.CreateCertificate(rand.Reader, tpl, parent, pub, by)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
x, _ := x509.ParseCertificate(der)
|
|
return x, string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}))
|
|
}
|
|
ca, caPEM := sign(&x509.Certificate{Subject: pkix.Name{CommonName: name + " Intermediate CA"}, IsCA: true, BasicConstraintsValid: true,
|
|
KeyUsage: x509.KeyUsageCertSign, NotBefore: now.Add(-3 * 365 * 24 * time.Hour), NotAfter: now.Add(caLeft)}, nil, &caKey.PublicKey, caKey)
|
|
v.issuer = caPEM
|
|
add := func(x *x509.Certificate, p string) {
|
|
s := serialHex(x)
|
|
v.certs[s], v.serials = p, append(v.serials, s)
|
|
}
|
|
add(ca, caPEM)
|
|
leafKey, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
|
day := 24 * time.Hour
|
|
for i := range services {
|
|
cn := fmt.Sprintf("svc-%03d.%s.internal", i, strings.TrimPrefix(name, "vault-"))
|
|
// Most a long way from their end; one of vault-prod's within a
|
|
// week, its renewal overdue.
|
|
left := time.Duration(25+(i*37)%340) * day
|
|
if name == "vault-prod" && i == 102 {
|
|
left = 6 * day
|
|
}
|
|
for k, end := range []time.Duration{left - 30*day, left} { // the one it replaced, then the renewal
|
|
x, p := sign(&x509.Certificate{Subject: pkix.Name{CommonName: cn}, DNSNames: []string{cn}, NotBefore: now.Add(end - 365*day), NotAfter: now.Add(end)},
|
|
ca, &leafKey.PublicKey, caKey)
|
|
if k == 0 && end < -8*day {
|
|
continue // tidied away
|
|
}
|
|
add(x, p)
|
|
}
|
|
}
|
|
for i := range short {
|
|
cn := fmt.Sprintf("job-%04d.batch.internal", i)
|
|
start := now.Add(-time.Duration(i%60) * time.Minute)
|
|
x, p := sign(&x509.Certificate{Subject: pkix.Name{CommonName: cn}, DNSNames: []string{cn}, NotBefore: start, NotAfter: start.Add(time.Hour)}, ca, &leafKey.PublicKey, caKey)
|
|
add(x, p)
|
|
}
|
|
if len(v.serials) > 5 {
|
|
v.revoked = []string{v.serials[3]}
|
|
}
|
|
return v
|
|
}
|
|
|
|
func serialHex(x *x509.Certificate) string {
|
|
b := x.SerialNumber.Bytes()
|
|
parts := make([]string, len(b))
|
|
for i, c := range b {
|
|
parts[i] = fmt.Sprintf("%02x", c)
|
|
}
|
|
return strings.Join(parts, ":")
|
|
}
|
|
|
|
func (v *fakeVault) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
p := strings.TrimPrefix(r.URL.Path, "/v1/")
|
|
list := r.Method == "LIST" || r.URL.Query().Get("list") == "true"
|
|
now := time.Now()
|
|
var out any
|
|
data := func(d any) any { return map[string]any{"data": d} }
|
|
switch {
|
|
case p == "sys/health":
|
|
mode := "disabled"
|
|
if v.enterprise {
|
|
mode = "primary"
|
|
}
|
|
out = map[string]any{"initialized": true, "sealed": false, "standby": false, "version": v.version, "cluster_name": v.name,
|
|
"enterprise": v.enterprise, "replication_dr_mode": mode, "replication_performance_mode": "disabled"}
|
|
case p == "sys/leader":
|
|
out = map[string]any{"ha_enabled": true, "is_self": true, "leader_address": "https://" + v.servers[0].id + "." + v.name + ":8200"}
|
|
case p == "sys/storage/raft/autopilot/state":
|
|
servers, voters, healthy, ft := map[string]any{}, []string{}, true, v.ft
|
|
for _, s := range v.servers {
|
|
if v.name == "vault-edge" && s.id == "e3" && spellVaultEdgeDown.active(now) {
|
|
s.node, s.status, ft = "failed", "voter", 0
|
|
}
|
|
ok := s.node == "alive"
|
|
healthy = healthy && ok
|
|
sv := map[string]any{"id": s.id, "name": v.name + "-" + s.id, "address": s.id + ":8201", "node_status": s.node, "healthy": ok,
|
|
"status": s.status, "last_contact": "0s", "version": strings.TrimSuffix(v.version, "+ent"), "stable_since": now.Add(-40 * 24 * time.Hour).Format(time.RFC3339)}
|
|
if !ok {
|
|
sv["last_contact"] = "41m12s"
|
|
}
|
|
servers[s.id] = sv
|
|
voters = append(voters, s.id)
|
|
}
|
|
out = data(map[string]any{"healthy": healthy, "failure_tolerance": ft, "voters": voters, "servers": servers})
|
|
case p == "auth/token/lookup-self":
|
|
out = data(map[string]any{"display_name": "token-obs-beacond", "policies": []string{"default", "obs-monitor"}, "ttl": 20 * 3600,
|
|
"creation_ttl": 24 * 3600, "renewable": true, "type": "service"})
|
|
case p == "auth/token/renew-self":
|
|
out = map[string]any{"auth": map[string]any{"lease_duration": 24 * 3600}}
|
|
case p == "sys/metrics":
|
|
l := vaultLeases(v.leases, now)
|
|
fmt.Fprintf(w, "# TYPE vault_expire_num_leases gauge\nvault_expire_num_leases %d\nvault_expire_num_irrevocable_leases 0\n", l)
|
|
if v.enterprise {
|
|
fmt.Fprint(w, "vault_audit_log_request_failure 0\nvault_audit_log_response_failure 0\n")
|
|
}
|
|
return
|
|
case p == "sys/mounts":
|
|
out = data(map[string]any{v.mount + "/": map[string]any{"type": "pki"}, "secret/": map[string]any{"type": "kv"}})
|
|
case p == "sys/replication/status" && v.enterprise:
|
|
out = data(map[string]any{"dr": map[string]any{"mode": "primary", "state": "running", "known_secondaries": []string{"dr-west"}, "last_wal": 912_334,
|
|
"secondaries": []any{map[string]any{"node_id": "dr-west", "api_address": "https://vault-dr.west:8200", "connection_state": "ready",
|
|
"last_heartbeat": now.Add(-4 * time.Second).Format(time.RFC3339)}}}, "performance": map[string]any{"mode": "disabled"}})
|
|
case p == "sys/license/status" && v.enterprise:
|
|
out = data(map[string]any{"autoloading_used": true, "autoloaded": map[string]any{"license_id": "fleet-0000",
|
|
"expiration_time": now.Add(300 * 24 * time.Hour).Format(time.RFC3339), "termination_time": now.Add(310 * 24 * time.Hour).Format(time.RFC3339)}})
|
|
case p == v.mount+"/issuers" && list:
|
|
out = data(map[string]any{"keys": []string{v.issuerID}})
|
|
case p == v.mount+"/issuer/"+v.issuerID:
|
|
out = data(map[string]any{"certificate": v.issuer, "issuer_name": "intermediate-2023"})
|
|
case p == v.mount+"/certs" && list:
|
|
out = data(map[string]any{"keys": v.serials})
|
|
case p == v.mount+"/certs/revoked" && list:
|
|
out = data(map[string]any{"keys": v.revoked})
|
|
case strings.HasPrefix(p, v.mount+"/cert/"):
|
|
c, ok := v.certs[strings.TrimPrefix(p, v.mount+"/cert/")]
|
|
if !ok {
|
|
w.WriteHeader(http.StatusNotFound)
|
|
out = map[string]any{"errors": []string{}}
|
|
break
|
|
}
|
|
out = data(map[string]any{"certificate": c})
|
|
default:
|
|
w.WriteHeader(http.StatusNotFound)
|
|
out = map[string]any{"errors": []string{}}
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(out)
|
|
}
|
|
|
|
// vaultLeaseBase is each fake Vault's usual number of leases.
|
|
var vaultLeaseBase = map[string]int{"vault-prod": 18_000, "vault-edge": 900}
|
|
|
|
// vaultLeases is how many leases a Vault holds at t: more in the working
|
|
// day, when people and jobs log in, with a little noise.
|
|
func vaultLeases(base int, t time.Time) int {
|
|
h := float64(t.UTC().Hour()) + float64(t.UTC().Minute())/60
|
|
day := math.Sin(2 * math.Pi * (h - 14) / 24) // UTC 20:00 is early afternoon in Phoenix
|
|
return base + int(float64(base)*(0.12*day+0.02*jitter("leases", t.Truncate(time.Minute))))
|
|
}
|