Add two storefronts, a payment gateway, and alerts worth waking up for
The estate could show an operator their infrastructure and had nothing to say to the business paying for it. Two storefronts and the gateway behind both fix that: Magento on two hosts, WooCommerce on one, and pay-01 carrying authorisations with amount, gateway and decline reason. Orders, revenue, average order value, where checkout loses people and why a card was refused now come out of the same log lines the operators are already reading, which is the argument for not running a separate metrics stack beside this one. Two platforms rather than one deliberately. Magento and WooCommerce write about the same events differently, so a panel that groups by service instead of assuming a single shape is the honest way to build one -- and the demo shows that rather than describing it. Order totals are built from a basket of real SKUs at real prices rather than drawn from a distribution, so average order value moves the way one actually moves. Declines rise during the seeded outage window alongside the 5xx rate, because whatever fails requests fails authorisations too. Twenty-seven new alert rules, thresholds calibrated against what the fleet actually emits -- measured on the demo's own week of history rather than guessed. A rule set at the average fires constantly and one set an order of magnitude above it never fires; these sit two to three times the steady-state rate, so they are quiet in normal operation and true during the diurnal peak or the seeded incident. Four are absence rules, because a domain controller or a storefront going silent is not a threshold question. Six new dashboards: fleet health, golden signals, security posture, capacity and storage, commerce, payments. Three limits of the query language found the hard way and worth writing down, because each was discovered by a panel failing rather than by reading: `dc()` does not exist -- the functions are count, sum, avg, min and max; `or` is not supported between structured filters, so a panel spanning tiers filters on the attribute they share and groups by service; and dashboards refuse raw SQL outright. The validator run over all 169 panels and 38 rules now checks every one of those, plus stages, viz types and comparators.
This commit is contained in:
@@ -679,6 +679,12 @@ func primaryRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.Log
|
||||
return exchangeRecord(h, t, r, c)
|
||||
case "smb":
|
||||
return smbRecord(h, t, r, c)
|
||||
case "magento":
|
||||
return magentoRecord(h, t, r, c)
|
||||
case "woocommerce":
|
||||
return wooRecord(h, t, r, c)
|
||||
case "payments":
|
||||
return paymentsRecord(h, t, r, c)
|
||||
default:
|
||||
return newRecord(h, h.service, t, logsv1.Severity_SEVERITY_INFO, "heartbeat", nil)
|
||||
}
|
||||
@@ -1057,3 +1063,236 @@ func smbRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogReco
|
||||
map[string]string{"share": share, "winevt.target_user": user, "event_kind": "share_denied"})
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Commerce.
|
||||
//
|
||||
// Two storefronts and the gateway behind both, because the questions a
|
||||
// business asks of its logs are not the questions an operator asks, and
|
||||
// a demo that only answers the second one is only half a demo. Orders,
|
||||
// revenue, average order value, where checkout is losing people and why
|
||||
// a card was declined are all in the log line rather than in a separate
|
||||
// metrics system -- which is the argument for keeping the two together.
|
||||
//
|
||||
// Two platforms rather than one on purpose: Magento and WooCommerce
|
||||
// write differently about the same events, so a panel that groups by
|
||||
// service rather than assuming one shape is the honest way to build one.
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
var (
|
||||
storeViews = []string{"uk", "us", "de", "fr"}
|
||||
|
||||
skus = []struct {
|
||||
sku, name string
|
||||
price float64
|
||||
}{
|
||||
{"CH-1042", "Aeron-style task chair", 489.00},
|
||||
{"DK-2201", "Standing desk 160x80", 629.00},
|
||||
{"MN-3310", "27\" 4K monitor", 379.99},
|
||||
{"KB-4407", "Mechanical keyboard, tactile", 129.50},
|
||||
{"MS-5120", "Vertical ergonomic mouse", 74.95},
|
||||
{"LT-6003", "Desk lamp, warm CCT", 59.00},
|
||||
{"CB-7788", "Cable management tray", 24.99},
|
||||
{"HS-8890", "Noise-cancelling headset", 219.00},
|
||||
}
|
||||
|
||||
checkoutSteps = []string{"cart", "shipping", "payment", "review", "placed"}
|
||||
|
||||
gateways = []string{"stripe", "adyen", "paypal"}
|
||||
|
||||
declineReasons = []struct {
|
||||
code, text string
|
||||
weight int
|
||||
}{
|
||||
{"insufficient_funds", "Insufficient funds", 30},
|
||||
{"do_not_honor", "Do not honour", 22},
|
||||
{"expired_card", "Expired card", 14},
|
||||
{"incorrect_cvc", "Incorrect CVC", 12},
|
||||
{"lost_or_stolen", "Lost or stolen card", 6},
|
||||
{"3ds_failed", "3-D Secure authentication failed", 16},
|
||||
}
|
||||
declineWeightTotal int
|
||||
|
||||
paymentMethods = []string{"card", "paypal", "apple_pay", "klarna"}
|
||||
)
|
||||
|
||||
func init() {
|
||||
for _, d := range declineReasons {
|
||||
declineWeightTotal += d.weight
|
||||
}
|
||||
}
|
||||
|
||||
func pickDecline(r *rand.Rand) (string, string) {
|
||||
n := r.Intn(declineWeightTotal)
|
||||
for _, d := range declineReasons {
|
||||
if n -= d.weight; n < 0 {
|
||||
return d.code, d.text
|
||||
}
|
||||
}
|
||||
return declineReasons[0].code, declineReasons[0].text
|
||||
}
|
||||
|
||||
// orderTotal builds a basket rather than drawing a number, so average
|
||||
// order value moves the way a real one does -- driven by what is in the
|
||||
// cart, not by a distribution somebody chose.
|
||||
func orderTotal(r *rand.Rand) (float64, int, string) {
|
||||
items := 1 + r.Intn(4)
|
||||
total := 0.0
|
||||
first := ""
|
||||
for i := 0; i < items; i++ {
|
||||
s := skus[r.Intn(len(skus))]
|
||||
qty := 1
|
||||
if r.Float64() < 0.18 {
|
||||
qty = 2
|
||||
}
|
||||
total += s.price * float64(qty)
|
||||
if i == 0 {
|
||||
first = s.sku
|
||||
}
|
||||
}
|
||||
return total, items, first
|
||||
}
|
||||
|
||||
func magentoRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord {
|
||||
store := pick(r, storeViews)
|
||||
switch n := r.Intn(100); {
|
||||
case n < 34:
|
||||
// Checkout progress. The funnel narrows towards `placed`, which is
|
||||
// what makes a "where are we losing people" panel say anything.
|
||||
step := checkoutSteps[0]
|
||||
switch f := r.Float64(); {
|
||||
case f < 0.34:
|
||||
step = "cart"
|
||||
case f < 0.58:
|
||||
step = "shipping"
|
||||
case f < 0.76:
|
||||
step = "payment"
|
||||
case f < 0.88:
|
||||
step = "review"
|
||||
default:
|
||||
step = "placed"
|
||||
}
|
||||
return newRecord(h, "magento", t, logsv1.Severity_SEVERITY_INFO,
|
||||
fmt.Sprintf("checkout step reached: %s quote_id=%d store=%s", step, 400000+r.Intn(99999), store),
|
||||
map[string]string{"checkout_step": step, "store_view": store, "event_kind": "checkout"})
|
||||
case n < 58:
|
||||
total, items, sku := orderTotal(r)
|
||||
return newRecord(h, "magento", t, logsv1.Severity_SEVERITY_INFO,
|
||||
fmt.Sprintf("Order placed: increment_id=%d grand_total=%.2f items=%d store=%s method=%s",
|
||||
2000000000+r.Intn(99999999), total, items, store, pick(r, paymentMethods)),
|
||||
map[string]string{
|
||||
"event_kind": "order", "order_total": fmt.Sprintf("%.2f", total),
|
||||
"order_items": strconv.Itoa(items), "sku": sku, "store_view": store,
|
||||
"currency": "GBP", "payment_method": pick(r, paymentMethods),
|
||||
})
|
||||
case n < 72:
|
||||
s := skus[r.Intn(len(skus))]
|
||||
return newRecord(h, "magento", t, logsv1.Severity_SEVERITY_INFO,
|
||||
fmt.Sprintf("Product viewed: sku=%s name=%q store=%s", s.sku, s.name, store),
|
||||
map[string]string{"event_kind": "product_view", "sku": s.sku, "store_view": store})
|
||||
case n < 82:
|
||||
idx := pick(r, []string{"catalog_product_price", "cataloginventory_stock", "catalogsearch_fulltext", "customer_grid"})
|
||||
dur := 4 + r.Intn(180)
|
||||
return newRecord(h, "magento", t, logsv1.Severity_SEVERITY_INFO,
|
||||
fmt.Sprintf("Index %s has been rebuilt successfully in %02d:%02d:%02d", idx, 0, dur/60, dur%60),
|
||||
map[string]string{"event_kind": "reindex", "indexer": idx, "duration_ms": strconv.Itoa(dur * 1000)})
|
||||
case n < 90:
|
||||
return newRecord(h, "magento", t, logsv1.Severity_SEVERITY_INFO,
|
||||
fmt.Sprintf("Cron group %s finished, %d jobs run", pick(r, []string{"default", "index", "consumers"}), 1+r.Intn(20)),
|
||||
map[string]string{"event_kind": "cron", "store_view": store})
|
||||
case n < 96:
|
||||
s := skus[r.Intn(len(skus))]
|
||||
return newRecord(h, "magento", t, logsv1.Severity_SEVERITY_WARN,
|
||||
fmt.Sprintf("Not enough items for sale: sku=%s requested=%d on_hand=%d", s.sku, 1+r.Intn(3), r.Intn(2)),
|
||||
map[string]string{"event_kind": "out_of_stock", "sku": s.sku, "store_view": store})
|
||||
default:
|
||||
return newRecord(h, "magento", t, logsv1.Severity_SEVERITY_ERROR,
|
||||
fmt.Sprintf("main.CRITICAL: Uncaught TypeError in %s: Argument #1 must be of type Quote, null given",
|
||||
pick(r, []string{"Magento/Quote/Model/QuoteManagement.php", "Magento/Checkout/Model/Session.php", "Magento/Sales/Model/Order.php"})),
|
||||
map[string]string{"event_kind": "exception", "store_view": store})
|
||||
}
|
||||
}
|
||||
|
||||
func wooRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord {
|
||||
switch n := r.Intn(100); {
|
||||
case n < 38:
|
||||
total, items, sku := orderTotal(r)
|
||||
status := pick(r, []string{"processing", "completed", "on-hold"})
|
||||
return newRecord(h, "woocommerce", t, logsv1.Severity_SEVERITY_INFO,
|
||||
fmt.Sprintf("Order #%d status changed to %s (total %.2f, %d items)", 30000+r.Intn(9999), status, total, items),
|
||||
map[string]string{
|
||||
"event_kind": "order", "order_status": status, "order_total": fmt.Sprintf("%.2f", total),
|
||||
"order_items": strconv.Itoa(items), "sku": sku, "currency": "GBP",
|
||||
"payment_method": pick(r, paymentMethods),
|
||||
})
|
||||
case n < 58:
|
||||
return newRecord(h, "woocommerce", t, logsv1.Severity_SEVERITY_INFO,
|
||||
fmt.Sprintf("REST API request: GET /wp-json/wc/v3/products?per_page=%d served in %dms", 10+r.Intn(90), 20+r.Intn(600)),
|
||||
map[string]string{"event_kind": "api", "duration_ms": strconv.Itoa(20 + r.Intn(600))})
|
||||
case n < 74:
|
||||
return newRecord(h, "woocommerce", t, logsv1.Severity_SEVERITY_INFO,
|
||||
fmt.Sprintf("Scheduled action completed: %s", pick(r, []string{"woocommerce_cleanup_sessions", "wc_admin_unsnooze_admin_notes", "woocommerce_scheduled_sales"})),
|
||||
map[string]string{"event_kind": "cron"})
|
||||
case n < 84:
|
||||
s := skus[r.Intn(len(skus))]
|
||||
return newRecord(h, "woocommerce", t, logsv1.Severity_SEVERITY_INFO,
|
||||
fmt.Sprintf("Stock reduced for %s: %d -> %d", s.sku, 5+r.Intn(40), r.Intn(5)),
|
||||
map[string]string{"event_kind": "stock", "sku": s.sku})
|
||||
case n < 93:
|
||||
return newRecord(h, "woocommerce", t, logsv1.Severity_SEVERITY_WARN,
|
||||
fmt.Sprintf("Checkout error: %s", pick(r, []string{
|
||||
"Invalid billing postcode", "Coupon \"WELCOME10\" has expired",
|
||||
"Shipping method not available for this address", "Session expired before payment",
|
||||
})),
|
||||
map[string]string{"event_kind": "checkout_error"})
|
||||
default:
|
||||
return newRecord(h, "woocommerce", t, logsv1.Severity_SEVERITY_ERROR,
|
||||
"PHP Fatal error: Allowed memory size of 268435456 bytes exhausted in class-wc-order.php",
|
||||
map[string]string{"event_kind": "exception"})
|
||||
}
|
||||
}
|
||||
|
||||
func paymentsRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord {
|
||||
gw := pick(r, gateways)
|
||||
total, _, _ := orderTotal(r)
|
||||
took := int(float64(90+r.Intn(900)) * c.latencyMult)
|
||||
|
||||
// Declines rise with the outage window: the same dependency trouble
|
||||
// that fails API requests fails authorisations, which is what makes
|
||||
// the decline-rate rule true at the same time as the 5xx one.
|
||||
declineRate := 0.075
|
||||
if c.apiErrorRate > 0 {
|
||||
declineRate = 0.28
|
||||
}
|
||||
switch {
|
||||
case r.Float64() < declineRate:
|
||||
code, text := pickDecline(r)
|
||||
return newRecord(h, "payments", t, logsv1.Severity_SEVERITY_WARN,
|
||||
fmt.Sprintf("authorization declined gateway=%s amount=%.2f currency=GBP reason=%s (%s) latency=%dms", gw, total, code, text, took),
|
||||
map[string]string{
|
||||
"event_kind": "authorization", "auth_result": "declined", "gateway": gw,
|
||||
"decline_reason": code, "amount": fmt.Sprintf("%.2f", total),
|
||||
"currency": "GBP", "duration_ms": strconv.Itoa(took),
|
||||
})
|
||||
case r.Float64() < 0.05:
|
||||
return newRecord(h, "payments", t, logsv1.Severity_SEVERITY_INFO,
|
||||
fmt.Sprintf("refund issued gateway=%s amount=%.2f currency=GBP reason=%s", gw, total/2, pick(r, []string{"customer_request", "item_returned", "duplicate_charge"})),
|
||||
map[string]string{"event_kind": "refund", "gateway": gw, "amount": fmt.Sprintf("%.2f", total/2), "currency": "GBP"})
|
||||
case r.Float64() < 0.02:
|
||||
return newRecord(h, "payments", t, logsv1.Severity_SEVERITY_ERROR,
|
||||
fmt.Sprintf("chargeback received gateway=%s amount=%.2f currency=GBP network_reason=fraud", gw, total),
|
||||
map[string]string{"event_kind": "chargeback", "gateway": gw, "amount": fmt.Sprintf("%.2f", total), "currency": "GBP"})
|
||||
case r.Float64() < 0.10:
|
||||
return newRecord(h, "payments", t, logsv1.Severity_SEVERITY_INFO,
|
||||
fmt.Sprintf("3-D Secure challenge issued gateway=%s amount=%.2f currency=GBP", gw, total),
|
||||
map[string]string{"event_kind": "3ds_challenge", "gateway": gw, "amount": fmt.Sprintf("%.2f", total), "currency": "GBP"})
|
||||
default:
|
||||
return newRecord(h, "payments", t, logsv1.Severity_SEVERITY_INFO,
|
||||
fmt.Sprintf("authorization approved gateway=%s amount=%.2f currency=GBP latency=%dms", gw, total, took),
|
||||
map[string]string{
|
||||
"event_kind": "authorization", "auth_result": "approved", "gateway": gw,
|
||||
"amount": fmt.Sprintf("%.2f", total), "currency": "GBP",
|
||||
"duration_ms": strconv.Itoa(took),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -580,6 +580,46 @@ var fleet = []host{
|
||||
agentVersion: agentVersion, sourceKind: "eventlog", sourceDetail: "channels=Security,System,Application",
|
||||
batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000,
|
||||
},
|
||||
{
|
||||
name: "shop-mag-01", service: "magento",
|
||||
os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64",
|
||||
cores: 16, memTotal: 32 << 30, diskTot: 400 << 30,
|
||||
ipv4: "10.0.9.11", ipv6: "2600:3c02::f03c:94ff:fe1a:9011",
|
||||
cpuBase: 51, memFrac: 0.69, diskFrac: 0.46,
|
||||
eventsPerMin: 26, systemPerMin: 0.9,
|
||||
agentVersion: agentVersion, sourceKind: "file", sourceDetail: "/var/www/shop/var/log/system.log",
|
||||
batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000,
|
||||
},
|
||||
{
|
||||
name: "shop-mag-02", service: "magento",
|
||||
os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64",
|
||||
cores: 16, memTotal: 32 << 30, diskTot: 400 << 30,
|
||||
ipv4: "10.0.9.12", ipv6: "2600:3c02::f03c:94ff:fe1a:9012",
|
||||
cpuBase: 48, memFrac: 0.67, diskFrac: 0.44,
|
||||
eventsPerMin: 24, systemPerMin: 0.9,
|
||||
agentVersion: agentVersion, sourceKind: "file", sourceDetail: "/var/www/shop/var/log/system.log",
|
||||
batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000,
|
||||
},
|
||||
{
|
||||
name: "shop-woo-01", service: "woocommerce",
|
||||
os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64",
|
||||
cores: 8, memTotal: 16 << 30, diskTot: 200 << 30,
|
||||
ipv4: "10.0.9.21", ipv6: "2600:3c02::f03c:94ff:fe1a:9021",
|
||||
cpuBase: 37, memFrac: 0.58, diskFrac: 0.39,
|
||||
eventsPerMin: 17, systemPerMin: 0.7,
|
||||
agentVersion: agentVersion, sourceKind: "file", sourceDetail: "/var/www/woo/wp-content/uploads/wc-logs/",
|
||||
batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000,
|
||||
},
|
||||
{
|
||||
name: "pay-01", service: "payments",
|
||||
os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64",
|
||||
cores: 8, memTotal: 16 << 30, diskTot: 120 << 30,
|
||||
ipv4: "10.0.9.31", ipv6: "2600:3c02::f03c:94ff:fe1a:9031",
|
||||
cpuBase: 23, memFrac: 0.47, diskFrac: 0.28,
|
||||
eventsPerMin: 19, systemPerMin: 0.6,
|
||||
agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=shop-payments.service",
|
||||
batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000,
|
||||
},
|
||||
{
|
||||
name: "legacy-01", service: "nginx",
|
||||
os: "Ubuntu 20.04.6 LTS", kernel: "5.4.0-192-generic", arch: "x86_64",
|
||||
|
||||
Reference in New Issue
Block a user