cairnobs/hack/demo-simulator/incidents.go

package main

import (
	"math"
	"time"
)

// Incidents are what turn a wall of uniform noise into a dataset worth
// clicking around in: a demo viewer who filters to 5xx, or opens the
// Alerts page, should find a *story* (one bad API node, a burst of SSH
// probing, a spam wave) rather than the same flat error rate everywhere.
// Every window below is expressed relative to `origin` -- the moment the
// simulator started, which is also the end of the backfill window -- so
// a freshly reset demo always has its incidents at the same, recent,
// predictable offsets no matter what day it is.
type conditions struct {
	// apiErrorRate replaces the API tier's baseline 5xx probability for
	// the affected host during an outage window.
	apiErrorRate float64
	// latencyMult multiplies API/DB latencies -- an outage that only
	// changed status codes without slowing anything down wouldn't look
	// like a real one.
	latencyMult float64
	// jobFailureRate replaces the worker tier's baseline failure
	// probability. An API/database outage doesn't stay in the request
	// path: the same failing dependencies take background jobs down with
	// them, which is also the only thing that ever makes the job-failure
	// alert rule true -- a steady 5% background failure rate is normal
	// operation, not an incident.
	jobFailureRate float64
	// bruteForce and spamWave switch the system/smtp generators from
	// their normal mix to an attack-shaped one for the window.
	bruteForce bool
	spamWave   bool
}

// Windows, as offsets back from origin. Kept as one table so the story
// is readable in one place and the alert rules in
// /hack/demo-seed/alerts can be written against known-true conditions.
const (
	apiOutageStart = 8 * time.Hour
	apiOutageEnd   = 6*time.Hour + 30*time.Minute
	apiOutageHost  = "api-02"

	bruteForceStart = 14 * time.Hour
	bruteForceEnd   = 13 * time.Hour

	spamWaveStart = 30 * time.Hour
	spamWaveEnd   = 26 * time.Hour

	// Live mode can't rely on the backfill windows above -- they recede
	// into the past as a demo session runs. These recurring bursts keep
	// the *present* interesting too, which is what the alert rules
	// (evaluating over -5m/-10m windows) actually see: without them,
	// every rule would settle into a permanent OK state a few minutes
	// after a reset and the Alerts page would never do anything again.
	liveErrorBurstPeriod = 47 * time.Minute
	liveErrorBurstLen    = 5 * time.Minute
	liveProbePeriod      = 2 * time.Hour
	liveProbeLen         = 8 * time.Minute
	liveSpamPeriod       = 3 * time.Hour
	liveSpamLen          = 10 * time.Minute
)

func conditionsAt(t, origin time.Time, h *host) conditions {
	hostName := h.name
	c := conditions{latencyMult: 1}
	since := origin.Sub(t)

	// Is an API-tier outage in effect? Two sources feed the same
	// handling: the backfilled incident window, and -- in live mode --
	// the recurring burst that keeps the present interesting.
	outage := 0.0
	if since <= apiOutageStart && since >= apiOutageEnd {
		outage = 0.42
	}
	bruteForce := since <= bruteForceStart && since >= bruteForceEnd
	spamWave := since <= spamWaveStart && since >= spamWaveEnd

	if t.After(origin) {
		// Phases are measured from origin so the first burst of each kind
		// lands a predictable few minutes into a demo session rather than
		// immediately at reset.
		elapsed := t.Sub(origin)
		if phase := (elapsed + 10*time.Minute) % liveErrorBurstPeriod; phase < liveErrorBurstLen {
			outage = 0.45
		}
		if phase := (elapsed + 20*time.Minute) % liveProbePeriod; phase < liveProbeLen {
			bruteForce = true
		}
		if phase := (elapsed + 35*time.Minute) % liveSpamPeriod; phase < liveSpamLen {
			spamWave = true
		}
	}

	if outage > 0 {
		switch {
		case hostName == apiOutageHost:
			c.apiErrorRate = outage
			c.latencyMult = 4.5
		case internetFacingName(hostName):
			// The edge tier fronts all three API nodes, so roughly a
			// third of what it proxies during the outage hits the failing
			// one. Without this the outage would be invisible from the
			// edge, which isn't how a viewer expects to be able to trace
			// it: client-visible 5xx are exactly what makes it an outage
			// rather than an internal blip.
			c.apiErrorRate = outage / 3
			c.latencyMult = 2
		case hostName == "db-01":
			// The database is the *cause*, not a second unrelated
			// incident -- a viewer who drills from the API errors into
			// the same window on db-01 should find slow queries waiting.
			c.latencyMult = 6
		case h.service == "worker":
			c.jobFailureRate = 0.35
			c.latencyMult = 2.5
		}
	}
	c.bruteForce = bruteForce
	c.spamWave = spamWave

	return c
}

// diurnal scales event rates by time of day: a shop's traffic peaks
// mid-afternoon UTC and bottoms out around 04:00, roughly a 3.5x spread.
// Without this every chart is a flat line and the "last 24h" view tells
// a viewer nothing that "last 1h" didn't.
func diurnal(t time.Time) float64 {
	hour := float64(t.UTC().Hour()) + float64(t.UTC().Minute())/60
	// Peak at 15:00 UTC, trough at 03:00.
	return 0.35 + 0.65*(0.5+0.5*math.Cos((hour-15)/24*2*math.Pi))
}