Demo: start with 30 days behind it #51
No files matched your search
@@ -46,6 +46,11 @@ obs-beacond and the simulator, so nothing in it can reach anything else:
|
||||
- `fleet config -demo` writes the server's pieces; `fleet run -demo` runs
|
||||
the fleet (add `-ibmi 4` for IBM i and the HMC). Start the fleet first so
|
||||
its stand-ins are up for obs-beacond's first readings.
|
||||
- `fleet run -history <database> -state-key <obs-beacond's state key>`
|
||||
first writes the last 30 days into obs-beacond's (SQLite) database, so
|
||||
the graphs, reports and incident history aren't empty on a fresh demo
|
||||
(below). It takes under a minute, before the stand-ins start, so wait
|
||||
for them that long; it leaves a database that already has history alone.
|
||||
- The root is read-only and the state is in tmpfs, so restarting the
|
||||
container is a fresh demo; restart it on a timer (weekly is plenty) so
|
||||
history doesn't pile up.
|
||||
@@ -59,3 +64,14 @@ problems that come and go through the day, so incidents open, resolve and
|
||||
show up under Recently resolved. Because the plan runs on the clock, two
|
||||
visitors an hour apart see different problems, and a restarted demo shows
|
||||
the same ones the old one would have.
|
||||
|
||||
The demo starts with a month behind it, worked out from the same plan and
|
||||
clock (`scripts/fleet/history.go`): every host's hourly CPU, memory and
|
||||
disks (a working day in its site's time zone, quieter weekends, memory
|
||||
that creeps until a weekly restart, disks that fill until a clean-up), the
|
||||
clusters' nodes (and a week of their controllers and VMs, to keep the
|
||||
database small), the arrays' I/O, the IBM i partitions' CPU, the
|
||||
Vaults' leases, the mail monitor's rounds, availability for the reports,
|
||||
and an incident for each time a problem came and went, with notes on the
|
||||
stories saying what the cause was. Its incident numbers carry on from
|
||||
there.
|
||||
@@ -82,6 +82,15 @@ restarted demo carries on where the clock is. Incidents open and resolve
|
||||
through the day, and a dozen or two are open at any moment. The mail
|
||||
server is healthy, to show what normal looks like.
|
||||
|
||||
What a host uses is a function of its name and the clock
|
||||
(`scripts/fleet/load.go`), not a dice roll, so `-history <database>
|
||||
-state-key <file>` can write the last 30 days (`-history-days`) before
|
||||
anything reports (`history.go`): hosts' and workloads' hours, mail rounds,
|
||||
statuses, and an incident for each past spell of the problems in
|
||||
`pastProblems` (scenario.go), whose keys and titles are obs-beacond's
|
||||
own. When you change what a spell's problem says in obs-beacond, change
|
||||
its line there too.
|
||||
|
||||
## Dependencies and notices
|
||||
|
||||
THIRD_PARTY_NOTICES.md lists every third-party component inside the
|
||||
|
||||
@@ -245,3 +245,53 @@ func (d *DB) Prune(before time.Time) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PutHostHours is PutHostHour for many, in one transaction: a backfill
|
||||
// (the demo's 30 days, scripts/fleet) or an import.
|
||||
func (d *DB) PutHostHours(hs []HostHour) error {
|
||||
t, err := d.begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer t.Rollback()
|
||||
for _, h := range hs {
|
||||
b, _ := json.Marshal(h.Disks)
|
||||
if _, err := t.Exec(`INSERT INTO host_hours (host, hour, cpu, cpu_max, mem, mem_max, disks) VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (host, hour) DO UPDATE SET cpu=excluded.cpu, cpu_max=excluded.cpu_max, mem=excluded.mem, mem_max=excluded.mem_max, disks=excluded.disks`,
|
||||
h.Host, h.Hour.Unix(), h.CPU, h.CPUMax, h.Mem, h.MemMax, string(b)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return t.Commit()
|
||||
}
|
||||
|
||||
// PutMailRounds is PutMailRound for many, in one transaction.
|
||||
func (d *DB) PutMailRounds(ms []MailRound) error {
|
||||
t, err := d.begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer t.Rollback()
|
||||
for _, m := range ms {
|
||||
if _, err := t.Exec(`INSERT INTO mail_rounds (at, seconds, queue) VALUES (?, ?, ?)
|
||||
ON CONFLICT (at) DO UPDATE SET seconds=excluded.seconds, queue=excluded.queue`, m.At.Unix(), m.Seconds, m.Queue); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return t.Commit()
|
||||
}
|
||||
|
||||
// LogStatuses is LogStatus for many, in one transaction.
|
||||
func (d *DB) LogStatuses(cs []Change) error {
|
||||
t, err := d.begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer t.Rollback()
|
||||
for _, c := range cs {
|
||||
if _, err := t.Exec(`INSERT INTO status_log (entity, at, status) VALUES (?, ?, ?)`, c.Entity, c.At.Unix(), c.Status); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return t.Commit()
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"math"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"github.com/cairnobs/cairnobs/internal/report"
|
||||
"github.com/cairnobs/cairnobs/internal/state"
|
||||
)
|
||||
|
||||
// The demo's past (#39): before anything reports, fleet run -history
|
||||
// writes the last 30 days into obs-beacond's database, worked out from the
|
||||
// same clock the live demo runs on (load.go, scenario.go), so the graphs
|
||||
// have their days and the Alerts page and reports have their incidents:
|
||||
//
|
||||
// - each host's hours: CPU and memory averages and peaks, disks;
|
||||
// - each cluster's workloads' hours, an array's I/O and space, an IBM i
|
||||
// partition's CPU, a Vault's leases;
|
||||
// - the mail monitor's rounds;
|
||||
// - hosts' and checks' statuses, from the start;
|
||||
// - an incident each time a past problem (pastProblems) came and went,
|
||||
// with a note from whoever looked at the stories.
|
||||
|
||||
// pastFlaps are the statuses that change with a spell, for the reports'
|
||||
// availability: an entity takes status while it's on, and after when
|
||||
// it's over.
|
||||
var pastFlaps = []struct {
|
||||
entity string
|
||||
s spell
|
||||
status, after string
|
||||
}{
|
||||
{"check/web-legacy", spellLegacy502, "warning", "good"},
|
||||
{"check/rbl-mx2", spellBlocklisted, "warning", "good"},
|
||||
{"svc/sites", spellLegacy502, "warning", "good"},
|
||||
{"svc/mail", spellBlocklisted, "warning", "good"},
|
||||
{"svc/postgres", spellReplicaBehind, "warning", "good"},
|
||||
{"svc/haproxy", spellReplicaBehind, "warning", "good"},
|
||||
// One copy of something there are several of is the service degraded,
|
||||
// not down (topology.go, redundant parts).
|
||||
{"svc/apps", spellAppExits, "warning", "good"},
|
||||
{"svc/redis", spellRedisFails, "warning", "good"},
|
||||
}
|
||||
|
||||
// pastFleet is what the history is written for.
|
||||
type pastFleet struct {
|
||||
hosts []*host
|
||||
clusters []*fakeCluster
|
||||
vaults bool // the Vault stand-ins run
|
||||
mail bool // the mail stand-in runs
|
||||
checks []string
|
||||
}
|
||||
|
||||
func writeHistory(path, keyFile string, days int, f pastFleet, now time.Time) error {
|
||||
start := time.Now()
|
||||
db, err := state.Open(path, keyFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
if !db.HistoryStart().IsZero() {
|
||||
log.Printf("fleet: %s already has history; left as it is", path)
|
||||
return nil
|
||||
}
|
||||
from := now.Truncate(time.Hour).Add(-time.Duration(days) * 24 * time.Hour)
|
||||
to := now.Truncate(time.Hour) // the hour now is obs-beacond's to write
|
||||
|
||||
var hs []state.HostHour
|
||||
var statuses []state.Change
|
||||
for _, h := range f.hosts {
|
||||
name := h.r.Host
|
||||
statuses = append(statuses, state.Change{Entity: "host/" + name, At: from, Status: "good"})
|
||||
for t := from; t.Before(to); t = t.Add(time.Hour) {
|
||||
hs = append(hs, hostHour(h, t))
|
||||
}
|
||||
}
|
||||
for _, id := range f.checks {
|
||||
statuses = append(statuses, state.Change{Entity: "check/" + id, At: from, Status: "good"})
|
||||
}
|
||||
if f.checks != nil { // the demo's services (demoTopology)
|
||||
for _, id := range []string{"shop", "api", "sites", "mail", "apps", "haproxy", "postgres", "redis", "nats"} {
|
||||
statuses = append(statuses, state.Change{Entity: "svc/" + id, At: from, Status: "good"})
|
||||
}
|
||||
// Flexible SSL on legacy.example is a lasting warning, and the
|
||||
// shop is behind Cloudflare.
|
||||
statuses = append(statuses, state.Change{Entity: "svc/cloudflare", At: from, Status: "warning"},
|
||||
state.Change{Entity: "svc/shop", At: from.Add(time.Second), Status: "warning"})
|
||||
for _, fl := range pastFlaps {
|
||||
s := fl.s
|
||||
for b := s.began(from).Add(s.every); b.Add(s.on).Before(now); b = b.Add(s.every) {
|
||||
statuses = append(statuses, state.Change{Entity: fl.entity, At: b.Add(20 * time.Second), Status: fl.status},
|
||||
state.Change{Entity: fl.entity, At: b.Add(s.on + 20*time.Second), Status: fl.after})
|
||||
}
|
||||
}
|
||||
slices.SortStableFunc(statuses, func(a, b state.Change) int { return a.At.Compare(b.At) })
|
||||
}
|
||||
if err := db.PutHostHours(hs); err != nil {
|
||||
return fmt.Errorf("host hours: %w", err)
|
||||
}
|
||||
if err := db.LogStatuses(statuses); err != nil {
|
||||
return fmt.Errorf("statuses: %w", err)
|
||||
}
|
||||
|
||||
nw := 0
|
||||
// Thousands of controllers and VMs get a week: a month of each is most
|
||||
// of the database, and the demo's state is a 256 MB tmpfs.
|
||||
few := to.Add(-time.Duration(min(days, 7)) * 24 * time.Hour)
|
||||
for _, c := range f.clusters {
|
||||
ws := clusterHours(c, from, few, to)
|
||||
if err := db.PutWorkloadHours(ws); err != nil {
|
||||
return fmt.Errorf("%s: %w", c.name, err)
|
||||
}
|
||||
nw += len(ws)
|
||||
}
|
||||
if f.vaults {
|
||||
var ws []state.WorkloadHour
|
||||
for name, base := range vaultLeaseBase {
|
||||
for t := from; t.Before(to); t = t.Add(time.Hour) {
|
||||
avg, top := 0.0, 0.0
|
||||
for m := range 4 {
|
||||
l := float64(vaultLeases(base, t.Add(time.Duration(m)*15*time.Minute)))
|
||||
avg += l / 4
|
||||
top = max(top, l)
|
||||
}
|
||||
ws = append(ws, state.WorkloadHour{Source: report.SourceKey("obs-beacond", name), Workload: name, Hour: t, Kind: report.KindCluster,
|
||||
Leases: math.Round(avg), LeasesMax: top})
|
||||
}
|
||||
}
|
||||
if err := db.PutWorkloadHours(ws); err != nil {
|
||||
return fmt.Errorf("vault: %w", err)
|
||||
}
|
||||
nw += len(ws)
|
||||
}
|
||||
|
||||
var ms []state.MailRound
|
||||
if f.mail {
|
||||
for t := from; t.Before(now.Add(-time.Minute)); t = t.Add(time.Minute) {
|
||||
k := int(t.Unix() / 60)
|
||||
ms = append(ms, state.MailRound{At: t, Seconds: 1.5 + float64(k*7919%4500)/1000, Queue: 3 + k%6})
|
||||
}
|
||||
if err := db.PutMailRounds(ms); err != nil {
|
||||
return fmt.Errorf("mail rounds: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
n, err := writePastIncidents(db, from, now)
|
||||
if err != nil {
|
||||
return fmt.Errorf("incidents: %w", err)
|
||||
}
|
||||
log.Printf("fleet: wrote %d days of history in %s: %d host hours, %d workload hours, %d mail rounds, %d incidents",
|
||||
days, time.Since(start).Round(time.Millisecond), len(hs), nw, len(ms), n)
|
||||
return nil
|
||||
}
|
||||
|
||||
// hostHour is one host's hour, from a reading every five minutes, as
|
||||
// obs-beacond's own (from every report) would roughly be.
|
||||
func hostHour(h *host, t time.Time) state.HostHour {
|
||||
name := h.r.Host
|
||||
out := state.HostHour{Host: name, Hour: t}
|
||||
const n = 12
|
||||
for k := range n {
|
||||
at := t.Add(time.Duration(k)*5*time.Minute + 2*time.Minute)
|
||||
c, m := cpuNow(name, at), memAt(name, at)
|
||||
out.CPU += c / n
|
||||
out.Mem += m / n
|
||||
out.CPUMax = max(out.CPUMax, c*(1.1+0.3*seed(name, at.Format("15"))))
|
||||
out.MemMax = max(out.MemMax, m+1.5)
|
||||
}
|
||||
out.CPU, out.CPUMax = round1(out.CPU), round1(min(out.CPUMax, 100))
|
||||
out.Mem, out.MemMax = round1(out.Mem), round1(min(out.MemMax, 100))
|
||||
end := t.Add(time.Hour - time.Minute)
|
||||
for i, d := range h.r.Disks {
|
||||
out.Disks = append(out.Disks, state.DiskSample{Mount: d.Mount, Used: uint64(float64(d.Size) * diskAt(name, i, end)), Total: d.Size})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func round1(f float64) float64 { return math.Round(f*10) / 10 }
|
||||
|
||||
// clusterHours is a cluster's workloads' hours, from two readings an hour,
|
||||
// keyed the way obs-beacond keeps them (its historyKey): from from for
|
||||
// the cluster itself and its nodes, from few for the rest.
|
||||
func clusterHours(c *fakeCluster, from, few, to time.Time) []state.WorkloadHour {
|
||||
if c.noUsage && c.usageFn == nil {
|
||||
return nil
|
||||
}
|
||||
src := report.SourceKey(c.name, c.source)
|
||||
c.mu.Lock()
|
||||
names := map[string]report.Workload{}
|
||||
for id, r := range c.recs {
|
||||
names[id] = report.Workload{Namespace: r.Namespace, Name: r.Name, Kind: r.Kind}
|
||||
}
|
||||
c.mu.Unlock()
|
||||
var out []state.WorkloadHour
|
||||
for t := from; t.Before(to); t = t.Add(time.Hour) {
|
||||
sum := map[string]*state.WorkloadHour{}
|
||||
const n = 2
|
||||
for k := range n {
|
||||
at := t.Add(time.Duration(k)*30*time.Minute + 10*time.Minute)
|
||||
us := c.usageFn
|
||||
if us == nil {
|
||||
us = c.usage
|
||||
}
|
||||
for _, u := range us(at) {
|
||||
r, ok := names[u.ID]
|
||||
if !ok || t.Before(few) && r.Kind != report.KindNode && r.Kind != report.KindCluster {
|
||||
continue
|
||||
}
|
||||
key := r.Name
|
||||
if r.Namespace != "" {
|
||||
key = r.Namespace + "/" + r.Name
|
||||
}
|
||||
h := sum[key]
|
||||
if h == nil {
|
||||
h = &state.WorkloadHour{Source: src, Workload: key, Hour: t, Kind: r.Kind}
|
||||
sum[key] = h
|
||||
}
|
||||
h.CPU += u.CPU / n
|
||||
h.Mem += float64(u.Mem) / n
|
||||
h.CPUMax, h.MemMax = max(h.CPUMax, u.CPU), max(h.MemMax, float64(u.Mem))
|
||||
h.CPUOf, h.MemOf = u.CPUOf, float64(u.MemOf)
|
||||
if io := u.IO; io != nil {
|
||||
h.ReadOps += io.ReadOps / n
|
||||
h.WriteOps += io.WriteOps / n
|
||||
h.ReadUs += io.ReadUs / n
|
||||
h.WriteUs += io.WriteUs / n
|
||||
h.ReadBps += io.ReadBps / n
|
||||
h.WriteBps += io.WriteBps / n
|
||||
h.ReadUsMax, h.WriteUsMax = max(h.ReadUsMax, io.ReadUs*1.4), max(h.WriteUsMax, io.WriteUs*1.4)
|
||||
h.Used, h.UsedOf = float64(io.Used), float64(io.UsedOf)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, h := range sum {
|
||||
out = append(out, *h)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// writePastIncidents writes an incident each time a past problem came and
|
||||
// went in [from, now), numbered in order from B-0001, and tells
|
||||
// obs-beacond where the numbers carry on.
|
||||
func writePastIncidents(db *state.DB, from, now time.Time) (int, error) {
|
||||
type occ struct {
|
||||
p past
|
||||
began time.Time
|
||||
first bool // the first of its kind on its day: it gets the note
|
||||
}
|
||||
var all []occ
|
||||
for _, p := range pastProblems {
|
||||
if p.s.every == 0 {
|
||||
continue
|
||||
}
|
||||
day := ""
|
||||
for b := p.s.began(from).Add(p.s.every); b.Add(p.s.on).Before(now); b = b.Add(p.s.every) {
|
||||
d := b.UTC().Format("2006-01-02")
|
||||
all = append(all, occ{p, b, d != day})
|
||||
day = d
|
||||
}
|
||||
}
|
||||
slices.SortStableFunc(all, func(a, b occ) int { return a.began.Compare(b.began) })
|
||||
for k, o := range all {
|
||||
p := o.p
|
||||
// Seen within a reading or two, told after alerts.for (2m), and
|
||||
// resolved within a minute of its end.
|
||||
opened := o.began.Add(time.Duration(10+seedInt(p.key, o.began, 40)) * time.Second)
|
||||
fired := opened.Add(2 * time.Minute)
|
||||
resolved := o.began.Add(p.s.on + time.Duration(15+seedInt(p.key+" end", o.began, 45))*time.Second)
|
||||
inc := state.Incident{ID: fmt.Sprintf("B-%04d", k+1), Key: p.key, Severity: p.sev, Title: p.title, Detail: p.detail,
|
||||
Host: p.host, Href: p.href, Opened: opened, Fired: fired, Notified: fired, Resolved: resolved}
|
||||
if p.sev == "critical" && p.s.on > 10*time.Minute {
|
||||
inc.AckBy, inc.AckAt = "On call", fired.Add(time.Duration(2+seedInt(p.key+" ack", o.began, 6))*time.Minute)
|
||||
}
|
||||
if err := db.PutIncident(inc); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if p.note != "" && o.first {
|
||||
if err := db.SetNote(inc.ID, state.Note{Text: p.note, By: "On call", At: resolved.Add(20 * time.Minute)}); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return len(all), db.SetMeta("incident_seq", fmt.Sprint(len(all)))
|
||||
}
|
||||
|
||||
// seedInt is 0 … n-1, fixed for a key and a moment.
|
||||
func seedInt(key string, t time.Time, n int) int {
|
||||
return int((jitter(key, t) + 1) / 2 * float64(n-1))
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/cairnobs/cairnobs/internal/state"
|
||||
)
|
||||
|
||||
// Hosts stay under the warnings (disks 80%, memory 90%), but fleet-0023's
|
||||
// log volume, which crosses 80% for a while every ten days (#40).
|
||||
func TestLoadStaysCalm(t *testing.T) {
|
||||
now := time.Date(2026, 11, 16, 5, 0, 0, 0, time.UTC)
|
||||
over := 0
|
||||
for i := range 150 {
|
||||
name := hostName(i)
|
||||
for h := range 30 * 24 {
|
||||
at := now.Add(-time.Duration(h) * time.Hour)
|
||||
if c := cpuNow(name, at); c <= 0 || c > 100 {
|
||||
t.Fatalf("%s at %s: CPU %.1f", name, at, c)
|
||||
}
|
||||
if m := memAt(name, at); m >= 90 {
|
||||
t.Fatalf("%s at %s: memory %.1f", name, at, m)
|
||||
}
|
||||
for k := range 2 {
|
||||
if d := diskAt(name, k, at); d >= 0.8 {
|
||||
if i != 23 || k != 1 {
|
||||
t.Fatalf("%s disk %d at %s: %.2f", name, k, at, d)
|
||||
}
|
||||
over++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if over == 0 || over > 30*24/3 {
|
||||
t.Errorf("fleet-0023's log volume is over 80%% for %d of 720 hours", over)
|
||||
}
|
||||
// Not all alike, as every disk at 33% was.
|
||||
if diskAt(hostName(1), 0, now) == diskAt(hostName(2), 0, now) {
|
||||
t.Error("two hosts' disks are the same")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteHistory(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
key := filepath.Join(dir, "state.key")
|
||||
if err := os.WriteFile(key, []byte(base64.StdEncoding.EncodeToString(make([]byte, 32))), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db := filepath.Join(dir, "beacon.db")
|
||||
hosts := []*host{newHost(1, hostName(1), 2, 10*time.Second), newHost(2, hostName(2), 2, 10*time.Second)}
|
||||
now := time.Now()
|
||||
if err := writeHistory(db, key, 3, pastFleet{hosts: hosts, vaults: true, mail: true, checks: []string{"web-legacy"}}, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, err := state.Open(db, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Close()
|
||||
hs, _ := d.HostHours(now.Add(-4*24*time.Hour), now)
|
||||
if len(hs) != 2*3*24 {
|
||||
t.Errorf("%d host hours", len(hs))
|
||||
}
|
||||
incs, _ := d.IncidentsBetween(now.Add(-4*24*time.Hour), now)
|
||||
if len(incs) < 20 || d.Meta("incident_seq") != fmt.Sprint(len(incs)) {
|
||||
t.Errorf("%d incidents, incident_seq %s", len(incs), d.Meta("incident_seq"))
|
||||
}
|
||||
for _, i := range incs {
|
||||
if i.Resolved.After(now) || !i.Resolved.After(i.Opened) {
|
||||
t.Fatalf("%s %s: opened %s, resolved %s", i.ID, i.Key, i.Opened, i.Resolved)
|
||||
}
|
||||
}
|
||||
if d.HistoryStart().IsZero() {
|
||||
t.Error("no statuses")
|
||||
}
|
||||
// A second run leaves it alone.
|
||||
if err := writeHistory(db, key, 3, pastFleet{hosts: hosts}, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if hs2, _ := d.HostHours(now.Add(-4*24*time.Hour), now); len(hs2) != len(hs) {
|
||||
t.Errorf("written twice: %d hours", len(hs2))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"hash/fnv"
|
||||
"math"
|
||||
"time"
|
||||
)
|
||||
|
||||
// What a fake host uses, worked out from its name and the clock alone, so
|
||||
// the live reports and the 30 days written before them (history.go)
|
||||
// agree: a working day in its site's time zone and quieter weekends,
|
||||
// memory that creeps on some hosts until a weekly restart, and disks that
|
||||
// fill slowly until someone cleans up.
|
||||
|
||||
// seed is 0 … 1, fixed for a host and what it's for.
|
||||
func seed(name, what string) float64 {
|
||||
h := fnv.New64a()
|
||||
h.Write([]byte(name))
|
||||
h.Write([]byte{0})
|
||||
h.Write([]byte(what))
|
||||
return float64(h.Sum64()%1_000_003) / 1_000_003
|
||||
}
|
||||
|
||||
// jitter is -1 … 1, fixed for a host and a moment (to the second), so a
|
||||
// reading looks like a reading and the same moment always reads the same.
|
||||
func jitter(name string, t time.Time) float64 {
|
||||
h := fnv.New64a()
|
||||
h.Write([]byte(name))
|
||||
var b [8]byte
|
||||
u := uint64(t.Unix())
|
||||
for i := range b {
|
||||
b[i] = byte(u >> (8 * i))
|
||||
}
|
||||
h.Write(b[:])
|
||||
return float64(h.Sum64()%2_000_001)/1_000_000 - 1
|
||||
}
|
||||
|
||||
// siteZone is the hours from UTC where a host's site is: the first site
|
||||
// in Phoenix, then the US east coast, London and Frankfurt in turn.
|
||||
func siteZone(name string) time.Duration {
|
||||
n := fleetIndex(-1, name)
|
||||
if n < 0 {
|
||||
return -7 * time.Hour // the Windows servers are at the first site
|
||||
}
|
||||
return []time.Duration{-7 * time.Hour, -5 * time.Hour, 0, time.Hour}[(n/siteSize)%4]
|
||||
}
|
||||
|
||||
// workday is 0 … 1: how busy the working day is at t, where the host is.
|
||||
// It peaks early afternoon, is near nothing at night, and a weekend is a
|
||||
// third of a weekday.
|
||||
func workday(name string, t time.Time) float64 {
|
||||
local := t.Add(siteZone(name)).UTC()
|
||||
h := float64(local.Hour()) + float64(local.Minute())/60
|
||||
d := 0.0
|
||||
if h > 7 && h < 20 {
|
||||
d = math.Sin(math.Pi * (h - 7) / 13)
|
||||
}
|
||||
if wd := local.Weekday(); wd == time.Saturday || wd == time.Sunday {
|
||||
d /= 3
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// cpuAt is a host's CPU use (percent) at t, without the moment's jitter.
|
||||
// A third of hosts run a nightly job at 02:00 for half an hour.
|
||||
func cpuAt(name string, t time.Time) float64 {
|
||||
base := 3 + 17*seed(name, "cpu")
|
||||
swing := base * (0.5 + 1.5*seed(name, "cpu swing"))
|
||||
cpu := base + swing*workday(name, t)
|
||||
if seed(name, "nightly") < 0.33 {
|
||||
local := t.Add(siteZone(name)).UTC()
|
||||
if local.Hour() == 2 && local.Minute() < 30 {
|
||||
cpu += 25 + 30*seed(name, "nightly size")
|
||||
}
|
||||
}
|
||||
return min(cpu, 97)
|
||||
}
|
||||
|
||||
// cpuNow is cpuAt with the moment's jitter, as one reading.
|
||||
func cpuNow(name string, t time.Time) float64 {
|
||||
return max(0.5, cpuAt(name, t)*(1+0.25*jitter(name, t)))
|
||||
}
|
||||
|
||||
// memAt is a host's memory use (percent) at t. Two hosts in five run
|
||||
// something that grows until it's restarted, once a week.
|
||||
func memAt(name string, t time.Time) float64 {
|
||||
m := 30 + 40*seed(name, "mem") + 5*workday(name, t)
|
||||
if seed(name, "leaks") < 0.4 {
|
||||
week := 7 * 24 * time.Hour
|
||||
phase := time.Duration(seed(name, "leak phase") * float64(week))
|
||||
frac := float64((t.Add(phase).UnixNano())%int64(week)) / float64(week)
|
||||
m += 14 * frac
|
||||
}
|
||||
return min(m, 88) // memory warns at 90
|
||||
}
|
||||
|
||||
// diskAt is how full (0 … 1) a host's disk k is at t: a level of its own,
|
||||
// growing slowly over a cycle of one to three months until a clean-up
|
||||
// brings it back. fleet-0023's log volume is the one that fills: from 58%
|
||||
// to 86% in ten days, so it warns now and then.
|
||||
func diskAt(name string, k int, t time.Time) float64 {
|
||||
what := []string{"root", "data"}[min(k, 1)]
|
||||
base := 0.12 + 0.48*seed(name, what)
|
||||
grow := 0.04 + 0.12*seed(name, what+" growth")
|
||||
cycle := time.Duration(30+60*seed(name, what+" cycle")) * 24 * time.Hour
|
||||
if name == hostName(23) && k == 1 {
|
||||
base, grow, cycle = 0.58, 0.28, 10*24*time.Hour
|
||||
}
|
||||
phase := time.Duration(seed(name, what+" phase") * float64(cycle))
|
||||
frac := float64(t.Add(phase).UnixNano()%int64(cycle)) / float64(cycle)
|
||||
return min(base+grow*frac, 0.95)
|
||||
}
|
||||
+86
-79
@@ -191,6 +191,9 @@ func run(args []string) {
|
||||
vaultDir := fs.String("vault", "", "serve two fake Vault clusters over HTTPS on loopback, and write their vault: list for obs-beacond.yaml to this directory's vault.yaml (\"\": none)")
|
||||
demo := fs.Bool("demo", false, "hosts report the demo's checks, units, backups and data layer, and fleet serves its Patroni cluster (fleet config -demo, same number of agents)")
|
||||
probe := fs.Bool("probe", true, "time how long a change in cluster-00 takes to show on every copy (needs dev_signin)")
|
||||
histDB := fs.String("history", "", "first write the last days into this obs-beacond database (SQLite; history.go), unless it has some already (\"\": none)")
|
||||
histKey := fs.String("state-key", "", "the database's state_key_file, for -history")
|
||||
histDays := fs.Int("history-days", 30, "how many days -history writes")
|
||||
_ = fs.Parse(args)
|
||||
|
||||
type agent struct{ host, token string }
|
||||
@@ -279,26 +282,83 @@ func run(args []string) {
|
||||
}
|
||||
var made, delivered, queued atomic.Int64
|
||||
log.Printf("fleet: %d agents every %s, %d containers each, to %s, for %s", len(agents), *interval, *containers, *servers, *dur)
|
||||
all := append(slices.Clone(agents), winAgents...)
|
||||
hosts := make([]*host, len(all))
|
||||
for i, a := range all {
|
||||
var h *host
|
||||
switch {
|
||||
case i >= len(agents): // a Windows server (windows.go)
|
||||
h = newWindowsHost(a.host, *interval)
|
||||
default:
|
||||
h = newHost(i, a.host, *containers, *interval)
|
||||
}
|
||||
if *demo && h.win == nil {
|
||||
h.demo = newDemoHost(fleetIndex(i, a.host), len(agents), demoAll)
|
||||
h.demo.setup(&h.r)
|
||||
}
|
||||
if *withMap && h.win == nil {
|
||||
h.m = fakeMap(i, a.host, h.r.Containers)
|
||||
h.certs, h.files = hostCerts(i, a.host)
|
||||
}
|
||||
hosts[i] = h
|
||||
}
|
||||
// The clusters, made now so the history can be written from them.
|
||||
type sim struct {
|
||||
c *fakeCluster
|
||||
probe bool
|
||||
}
|
||||
var sims []sim
|
||||
for _, a := range clusterAgents {
|
||||
sims = append(sims, sim{newCluster(a.host, a.token, shape), a.host == "cluster-00"})
|
||||
}
|
||||
if *pveGuests > 0 && len(agents) > proxmoxHost {
|
||||
a := agents[proxmoxHost]
|
||||
sims = append(sims, sim{c: newProxmox(a.host, a.token, *pveGuests)})
|
||||
log.Printf("fleet: Proxmox cluster %s, %d nodes, %d guests, read by %s", proxmoxCluster, proxmoxNodes, *pveGuests, a.host)
|
||||
}
|
||||
if *vsphereVMs > 0 && len(agents) > vcenterHost {
|
||||
a := agents[vcenterHost]
|
||||
sims = append(sims, sim{c: newVSphere(a.host, a.token, *vsphereVMs)})
|
||||
log.Printf("fleet: vCenter %s, %d ESXi hosts, %d VMs, read by %s", vcenterName, esxiHosts, *vsphereVMs, a.host)
|
||||
}
|
||||
if *ntnxVMs > 0 && len(agents) > nutanixHost {
|
||||
a := agents[nutanixHost]
|
||||
sims = append(sims, sim{c: newNutanix(a.host, a.token, *ntnxVMs)})
|
||||
log.Printf("fleet: Prism Central %s, %d clusters of %d AHV hosts, %d VMs, read by %s", prismCentral, ntnxClusters, ahvPerPE, *ntnxVMs, a.host)
|
||||
}
|
||||
if n := min(*pureN, len(pureArrays)); n > 0 && len(agents) > pureHost {
|
||||
a := agents[pureHost]
|
||||
for k := range n {
|
||||
sims = append(sims, sim{c: newPure(a.host, a.token, k)})
|
||||
}
|
||||
log.Printf("fleet: %d Pure arrays (%v), read by %s", n, pureArrays[:n], a.host)
|
||||
}
|
||||
if n := min(*ibmiN, len(ibmiPartitions)); n > 0 && len(agents) > ibmiHost {
|
||||
a := agents[ibmiHost]
|
||||
for k := range n {
|
||||
sims = append(sims, sim{c: newIBMi(a.host, a.token, k)})
|
||||
}
|
||||
sims = append(sims, sim{c: newHMC(a.host, a.token)})
|
||||
log.Printf("fleet: %d IBM i partitions (%v) and the HMC %s, read by %s", n, ibmiPartitions[:n], hmcName, a.host)
|
||||
}
|
||||
if *histDB != "" {
|
||||
past := pastFleet{hosts: hosts, vaults: *vaultDir != "", mail: *mailDir != ""}
|
||||
for _, s := range sims {
|
||||
past.clusters = append(past.clusters, s.c)
|
||||
}
|
||||
for _, c := range demoAll {
|
||||
past.checks = append(past.checks, c.id)
|
||||
}
|
||||
if err := writeHistory(*histDB, *histKey, *histDays, past, time.Now()); err != nil {
|
||||
log.Fatalf("fleet: history: %v", err)
|
||||
}
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
for i, a := range append(slices.Clone(agents), winAgents...) {
|
||||
for i, a := range all {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
var h *host
|
||||
switch {
|
||||
case i >= len(agents): // a Windows server (windows.go)
|
||||
h = newWindowsHost(a.host, *interval)
|
||||
default:
|
||||
h = newHost(i, a.host, *containers, *interval)
|
||||
}
|
||||
if *demo && h.win == nil {
|
||||
h.demo = newDemoHost(fleetIndex(i, a.host), len(agents), demoAll)
|
||||
h.demo.setup(&h.r)
|
||||
}
|
||||
if *withMap && h.win == nil {
|
||||
h.m = fakeMap(i, a.host, h.r.Containers)
|
||||
h.certs, h.files = hostCerts(i, a.host)
|
||||
}
|
||||
h := hosts[i]
|
||||
// Agents start at random points in the interval, as a fleet does.
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
@@ -362,36 +422,17 @@ func run(args []string) {
|
||||
log.Fatalf("fleet: %v", err)
|
||||
}
|
||||
}
|
||||
for _, a := range clusterAgents {
|
||||
for _, s := range sims {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
c := newCluster(a.host, a.token, shape)
|
||||
var probed func(string, time.Time)
|
||||
if cookies != nil && a.host == "cluster-00" {
|
||||
if cookies != nil && s.probe {
|
||||
probed = func(word string, at time.Time) { watchProbe(ctx, client, urls, cookies, word, at, shown) }
|
||||
}
|
||||
runCluster(ctx, c, client, pick, batches, probed)
|
||||
runCluster(ctx, s.c, client, pick, batches, probed)
|
||||
}()
|
||||
}
|
||||
if *pveGuests > 0 && len(agents) > proxmoxHost {
|
||||
a := agents[proxmoxHost]
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
runCluster(ctx, newProxmox(a.host, a.token, *pveGuests), client, pick, batches, nil)
|
||||
}()
|
||||
log.Printf("fleet: Proxmox cluster %s, %d nodes, %d guests, read by %s", proxmoxCluster, proxmoxNodes, *pveGuests, a.host)
|
||||
}
|
||||
if *vsphereVMs > 0 && len(agents) > vcenterHost {
|
||||
a := agents[vcenterHost]
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
runCluster(ctx, newVSphere(a.host, a.token, *vsphereVMs), client, pick, batches, nil)
|
||||
}()
|
||||
log.Printf("fleet: vCenter %s, %d ESXi hosts, %d VMs, read by %s", vcenterName, esxiHosts, *vsphereVMs, a.host)
|
||||
}
|
||||
if *vaultDir != "" {
|
||||
serveVaults(ctx, *vaultDir)
|
||||
}
|
||||
@@ -407,42 +448,6 @@ func run(args []string) {
|
||||
if *akamaiDir != "" {
|
||||
serveAkamai(ctx, *akamaiDir)
|
||||
}
|
||||
if *ntnxVMs > 0 && len(agents) > nutanixHost {
|
||||
a := agents[nutanixHost]
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
runCluster(ctx, newNutanix(a.host, a.token, *ntnxVMs), client, pick, batches, nil)
|
||||
}()
|
||||
log.Printf("fleet: Prism Central %s, %d clusters of %d AHV hosts, %d VMs, read by %s", prismCentral, ntnxClusters, ahvPerPE, *ntnxVMs, a.host)
|
||||
}
|
||||
if n := min(*pureN, len(pureArrays)); n > 0 && len(agents) > pureHost {
|
||||
a := agents[pureHost]
|
||||
for k := range n {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
runCluster(ctx, newPure(a.host, a.token, k), client, pick, batches, nil)
|
||||
}()
|
||||
}
|
||||
log.Printf("fleet: %d Pure arrays (%v), read by %s", n, pureArrays[:n], a.host)
|
||||
}
|
||||
if n := min(*ibmiN, len(ibmiPartitions)); n > 0 && len(agents) > ibmiHost {
|
||||
a := agents[ibmiHost]
|
||||
for k := range n {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
runCluster(ctx, newIBMi(a.host, a.token, k), client, pick, batches, nil)
|
||||
}()
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
runCluster(ctx, newHMC(a.host, a.token), client, pick, batches, nil)
|
||||
}()
|
||||
log.Printf("fleet: %d IBM i partitions (%v) and the HMC %s, read by %s", n, ibmiPartitions[:n], hmcName, a.host)
|
||||
}
|
||||
if *ui > 0 && len(agents) > 0 {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
@@ -726,13 +731,15 @@ func (h *host) next(now time.Time) report.Report {
|
||||
r := h.r
|
||||
r.Time = now.UTC()
|
||||
r.Uptime = now.Sub(h.start).Seconds()
|
||||
r.CPU = 5 + 30*mrand.Float64()
|
||||
r.Load = [3]float64{r.CPU / 25, r.CPU / 30, r.CPU / 35}
|
||||
r.Mem.Available = uint64(float64(r.Mem.Total) * (0.3 + 0.4*mrand.Float64()))
|
||||
r.Net = report.Net{RxBps: 1e5 * mrand.Float64(), TxBps: 5e4 * mrand.Float64()}
|
||||
r.CPU = math.Round(cpuNow(r.Host, now)*10) / 10 // load.go
|
||||
cores := float64(max(r.CPUs, 1))
|
||||
r.Load = [3]float64{cores * r.CPU / 100, cores * cpuAt(r.Host, now.Add(-5*time.Minute)) / 100, cores * cpuAt(r.Host, now.Add(-15*time.Minute)) / 100}
|
||||
r.Mem.Available = uint64(float64(r.Mem.Total) * (1 - memAt(r.Host, now)/100))
|
||||
busy := 0.2 + workday(r.Host, now)
|
||||
r.Net = report.Net{RxBps: 4e5 * busy * (1 + 0.3*jitter(r.Host, now)), TxBps: 1.5e5 * busy * (1 + 0.3*jitter(r.Host+" tx", now))}
|
||||
disks := slices.Clone(r.Disks)
|
||||
for i := range disks {
|
||||
disks[i].Used = disks[i].Size / 3
|
||||
disks[i].Used = uint64(float64(disks[i].Size) * diskAt(r.Host, i, now))
|
||||
disks[i].Avail = disks[i].Size - disks[i].Used
|
||||
}
|
||||
r.Disks = disks
|
||||
|
||||
@@ -89,3 +89,45 @@ var (
|
||||
spellFSCrash = spell{every: 8 * time.Hour, at: 7 * time.Hour, on: time.Hour}
|
||||
spellActivation = spell{every: 3 * time.Hour, at: 20 * time.Minute, on: 80 * time.Minute}
|
||||
)
|
||||
|
||||
// past is a problem as obs-beacond titles it, for the incidents the demo's
|
||||
// history starts with (history.go): one for each time its spell came and
|
||||
// went before the demo started. The keys and titles are obs-beacond's
|
||||
// own, so the past and the present read alike.
|
||||
type past struct {
|
||||
s spell
|
||||
key, sev, title, host string
|
||||
href, detail, note string // note: what someone wrote afterwards, on the first of the stories' incidents in a day
|
||||
}
|
||||
|
||||
var pastProblems = []past{
|
||||
{s: spellLegacyDC, key: "akamai/property/legacy.corp.example.akadns.net", sev: "critical", href: "/akamai",
|
||||
title: "Akamai GTM: legacy.corp.example.akadns.net has no live datacenter",
|
||||
detail: "Every datacenter's server is failing the liveness test, so GTM answers with all of them, or none.",
|
||||
note: "Ashburn lost its upstream link for half an hour. The legacy app has one datacenter, so GTM had nowhere else to send it."},
|
||||
{s: spellLegacyDC, key: "akamai/traffic/400019", sev: "critical", href: "/akamai",
|
||||
title: "Akamai: 30% of CP code 400019's requests are failing with 5xx",
|
||||
detail: "Edge errors for the legacy property: its origin is the Ashburn server GTM can't reach."},
|
||||
{s: spellLegacyDC, key: "akamai/server/legacy.corp.example.akadns.net/198.51.100.50", sev: "warning", href: "/akamai",
|
||||
title: "198.51.100.50 in Ashburn is failing GTM's liveness test for legacy.corp.example.akadns.net"},
|
||||
{s: spellReplicaBehind, key: "cloudflare/errors/shop.example", sev: "warning", href: "/cloudflare",
|
||||
title: "shop.example: 7% of requests failing",
|
||||
detail: "Most on checkout.shop.example: 503s while HAProxy had one PostgreSQL replica for the shop's reads.",
|
||||
note: "fleet-0032 fell 40 MB behind during the nightly vacuum and HAProxy took it out; the one replica left couldn't carry checkout's reads. Moving the vacuum out of business hours."},
|
||||
{s: spellAppExits, key: "container/fleet-0007/app-04", sev: "critical", host: "fleet-0007", href: "/containers",
|
||||
title: "app-04 on fleet-0007: not running"},
|
||||
{s: spellDRPaths, key: "cluster/fleet-0040:fa-dr-01/connection/fa-prod-01", sev: "critical", host: "fleet-0040:fa-dr-01", href: "/clusters",
|
||||
title: "Replication to fa-prod-01 in fa-dr-01 (fleet-0040): connected, but its paths aren't redundant"},
|
||||
{s: spellLBFlaps, key: "cloudflare/origin/site-1-web/fleet-0052", sev: "warning", host: "fleet-0052", href: "/cloudflare",
|
||||
title: "fleet-0052 is failing site-1-web's health check"},
|
||||
{s: spellLBFlaps, key: "akamai/server/api.shop.example.akadns.net/203.0.2.11", sev: "warning", host: "fleet-0101", href: "/akamai",
|
||||
title: "203.0.2.11 in Singapore is failing GTM's liveness test for api.shop.example.akadns.net"},
|
||||
{s: spellVaultEdgeDown, key: "node/obs-beacond:vault-edge/vault-edge-e3", sev: "critical", host: "obs-beacond:vault-edge",
|
||||
href: "/workloads?source=obs-beacond%3Avault-edge&q=vault-edge-e3", title: "Server vault-edge-e3 in vault-edge (obs-beacond) is down"},
|
||||
{s: spellCrashloop, key: "workload/cluster-00/ns-07/deployment/app-097", sev: "warning", host: "cluster-00",
|
||||
href: "/workloads?source=cluster-00&ns=ns-07&q=app-097", title: "deployment/app-097 in ns-07 (cluster-00): 1 of 10 pods: 1 restarting (CrashLoopBackOff)"},
|
||||
{s: spellLegacy502, key: "check/web-legacy", sev: "warning", href: "/checks", title: "legacy.example.com is down"},
|
||||
{s: spellBlocklisted, key: "check/rbl-mx2", sev: "warning", href: "/checks", title: "mx2.example.com on block lists is down"},
|
||||
{s: spellLateBackup, key: "backup/fleet-0050/received/fleet-0080/pg-dump", sev: "serious", host: "fleet-0050", href: "/backups",
|
||||
title: "fleet-0080's pg-dump backup on fleet-0050 is 26 hours old"},
|
||||
}
|
||||
+14
-3
@@ -74,9 +74,9 @@ func serveVaults(ctx context.Context, dir string) {
|
||||
now := time.Now()
|
||||
vaults := []*fakeVault{
|
||||
newFakeVault("vault-prod", "1.21.4+ent", true, "pki_int", now, 300*24*time.Hour, 400, 2000,
|
||||
[]fakeRaft{{"v1", "leader", "alive"}, {"v2", "voter", "alive"}, {"v3", "voter", "alive"}}, 1, 18_000),
|
||||
[]fakeRaft{{"v1", "leader", "alive"}, {"v2", "voter", "alive"}, {"v3", "voter", "alive"}}, 1, vaultLeaseBase["vault-prod"]),
|
||||
newFakeVault("vault-edge", "2.4.1", false, "pki_edge", now, 12*24*time.Hour, 40, 0,
|
||||
[]fakeRaft{{"e1", "leader", "alive"}, {"e2", "voter", "alive"}, {"e3", "voter", "alive"}}, 1, 900),
|
||||
[]fakeRaft{{"e1", "leader", "alive"}, {"e2", "voter", "alive"}, {"e3", "voter", "alive"}}, 1, vaultLeaseBase["vault-edge"]),
|
||||
}
|
||||
var yaml strings.Builder
|
||||
yaml.WriteString("vault:\n")
|
||||
@@ -230,7 +230,7 @@ func (v *fakeVault) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
case p == "auth/token/renew-self":
|
||||
out = map[string]any{"auth": map[string]any{"lease_duration": 24 * 3600}}
|
||||
case p == "sys/metrics":
|
||||
l := v.leases + int(float64(v.leases)/10*math.Sin(float64(now.Unix())/3600))
|
||||
l := vaultLeases(v.leases, now)
|
||||
fmt.Fprintf(w, "# TYPE vault_expire_num_leases gauge\nvault_expire_num_leases %d\nvault_expire_num_irrevocable_leases 0\n", l)
|
||||
if v.enterprise {
|
||||
fmt.Fprint(w, "vault_audit_log_request_failure 0\nvault_audit_log_response_failure 0\n")
|
||||
@@ -268,3 +268,14 @@ func (v *fakeVault) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(out)
|
||||
}
|
||||
|
||||
// vaultLeaseBase is each fake Vault's usual number of leases.
|
||||
var vaultLeaseBase = map[string]int{"vault-prod": 18_000, "vault-edge": 900}
|
||||
|
||||
// vaultLeases is how many leases a Vault holds at t: more in the working
|
||||
// day, when people and jobs log in, with a little noise.
|
||||
func vaultLeases(base int, t time.Time) int {
|
||||
h := float64(t.UTC().Hour()) + float64(t.UTC().Minute())/60
|
||||
day := math.Sin(2 * math.Pi * (h - 14) / 24) // UTC 20:00 is early afternoon in Phoenix
|
||||
return base + int(float64(base)*(0.12*day+0.02*jitter("leases", t.Truncate(time.Minute))))
|
||||
}
|
||||
Reference in new issue
Block a user