coffey-labs/cairnobs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add two storefronts, a payment gateway, and alerts worth waking up for

Browse Source 
The estate could show an operator their infrastructure and had nothing to
say to the business paying for it. Two storefronts and the gateway behind
both fix that: Magento on two hosts, WooCommerce on one, and pay-01
carrying authorisations with amount, gateway and decline reason. Orders,
revenue, average order value, where checkout loses people and why a card
was refused now come out of the same log lines the operators are already
reading, which is the argument for not running a separate metrics stack
beside this one.

Two platforms rather than one deliberately. Magento and WooCommerce write
about the same events differently, so a panel that groups by service
instead of assuming a single shape is the honest way to build one -- and
the demo shows that rather than describing it.

Order totals are built from a basket of real SKUs at real prices rather
than drawn from a distribution, so average order value moves the way one
actually moves. Declines rise during the seeded outage window alongside
the 5xx rate, because whatever fails requests fails authorisations too.

Twenty-seven new alert rules, thresholds calibrated against what the
fleet actually emits -- measured on the demo's own week of history rather
than guessed. A rule set at the average fires constantly and one set an
order of magnitude above it never fires; these sit two to three times the
steady-state rate, so they are quiet in normal operation and true during
the diurnal peak or the seeded incident. Four are absence rules, because
a domain controller or a storefront going silent is not a threshold
question.

Six new dashboards: fleet health, golden signals, security posture,
capacity and storage, commerce, payments.

Three limits of the query language found the hard way and worth writing
down, because each was discovered by a panel failing rather than by
reading: `dc()` does not exist -- the functions are count, sum, avg, min
and max; `or` is not supported between structured filters, so a panel
spanning tiers filters on the attribute they share and groups by service;
and dashboards refuse raw SQL outright. The validator run over all 169
panels and 38 rules now checks every one of those, plus stages, viz types
and comparators.
This commit is contained in:
jcoffey-dev
2026-09-04 16:08:21 -07:00
parent f5ff19327b
commit f6c228b87c
36 changed files with 1466 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hack/demo-seed/dashboards/capacity-storage.json
+106
View File
@@ -0,0 +1,106 @@
{
"name": "Capacity and storage",
"description": "Disk, memory and the jobs that protect them -- what is filling up and what is backing it up",
"default_earliest": "-7d",
"default_latest": "now",
"panels": [
{
"title": "Peak disk used",
"query": "cairnobs.metrics=true | stats max(disk_used_bytes) as used",
"viz_type": "single_stat",
"position_x": 0,
"position_y": 0,
"width": 3,
"height": 3,
"query_language": "spl",
"sort_order": 0
},
{
"title": "Backup jobs",
"query": "service=backup event_kind=backup | stats count",
"viz_type": "single_stat",
"position_x": 3,
"position_y": 0,
"width": 3,
"height": 3,
"query_language": "spl",
"sort_order": 1
},
{
"title": "Backup failures",
"query": "service=backup result=error | stats count",
"viz_type": "single_stat",
"position_x": 6,
"position_y": 0,
"width": 3,
"height": 3,
"query_language": "spl",
"sort_order": 2
},
{
"title": "SQL log backups",
"query": "service=mssql event_kind=log_backup | stats count",
"viz_type": "single_stat",
"position_x": 9,
"position_y": 0,
"width": 3,
"height": 3,
"query_language": "spl",
"sort_order": 3
},
{
"title": "Disk used by host",
"query": "cairnobs.metrics=true | stats max(disk_used_bytes) as used by host | sort -used",
"viz_type": "bar",
"position_x": 0,
"position_y": 3,
"width": 12,
"height": 6,
"viz_config": {
"x_column": "host",
"value_column": "used"
},
"query_language": "spl",
"sort_order": 4
},
{
"title": "Memory headroom",
"query": "cairnobs.metrics=true | stats max(mem_used_bytes) as peak, max(mem_total_bytes) as total by host | sort -peak | head 15",
"viz_type": "table",
"position_x": 0,
"position_y": 9,
"width": 6,
"height": 5,
"query_language": "spl",
"sort_order": 5
},
{
"title": "Backup outcomes by job",
"query": "service=backup | stats count by backup_job, result",
"viz_type": "bar",
"position_x": 6,
"position_y": 9,
"width": 6,
"height": 5,
"viz_config": {
"x_column": "backup_job",
"value_column": "count",
"series_column": "result",
"stacked": "true"
},
"query_language": "spl",
"sort_order": 6
},
{
"title": "Recent backup failures",
"query": "service=backup result=error | sort -timestamp | head 20 | fields timestamp, host, backup_job, message",
"viz_type": "table",
"position_x": 0,
"position_y": 14,
"width": 12,
"height": 5,
"query_language": "spl",
"sort_order": 7
}
]
}
hack/demo-seed/dashboards/commerce.json
+164
View File
@@ -0,0 +1,164 @@
{
"name": "Commerce",
"description": "Orders, revenue and the checkout funnel across both storefronts",
"default_earliest": "-24h",
"default_latest": "now",
"panels": [
{
"title": "Orders",
"query": "event_kind=order | stats count",
"viz_type": "single_stat",
"position_x": 0,
"position_y": 0,
"width": 3,
"height": 3,
"query_language": "spl",
"sort_order": 0
},
{
"title": "Revenue",
"query": "event_kind=order | stats sum(order_total) as revenue",
"viz_type": "single_stat",
"position_x": 3,
"position_y": 0,
"width": 3,
"height": 3,
"query_language": "spl",
"sort_order": 1
},
{
"title": "Average order value",
"query": "event_kind=order | stats avg(order_total) as aov",
"viz_type": "single_stat",
"position_x": 6,
"position_y": 0,
"width": 3,
"height": 3,
"query_language": "spl",
"sort_order": 2
},
{
"title": "Items per order",
"query": "event_kind=order | stats avg(order_items) as items",
"viz_type": "single_stat",
"position_x": 9,
"position_y": 0,
"width": 3,
"height": 3,
"query_language": "spl",
"sort_order": 3
},
{
"title": "Revenue by storefront",
"query": "event_kind=order | stats sum(order_total) as revenue by service | sort -revenue",
"viz_type": "bar",
"position_x": 0,
"position_y": 3,
"width": 6,
"height": 5,
"viz_config": {
"x_column": "service",
"value_column": "revenue"
},
"query_language": "spl",
"sort_order": 4
},
{
"title": "Orders by store view",
"query": "service=magento event_kind=order | stats count by store_view | sort -count",
"viz_type": "bar",
"position_x": 6,
"position_y": 3,
"width": 6,
"height": 5,
"viz_config": {
"x_column": "store_view",
"value_column": "count"
},
"query_language": "spl",
"sort_order": 5
},
{
"title": "Checkout funnel",
"query": "service=magento event_kind=checkout | stats count by checkout_step | sort -count",
"viz_type": "bar",
"position_x": 0,
"position_y": 8,
"width": 6,
"height": 5,
"viz_config": {
"x_column": "checkout_step",
"value_column": "count"
},
"query_language": "spl",
"sort_order": 6
},
{
"title": "Payment methods",
"query": "event_kind=order | stats count by payment_method | sort -count",
"viz_type": "top_n",
"position_x": 6,
"position_y": 8,
"width": 6,
"height": 5,
"viz_config": {
"label_column": "payment_method",
"value_column": "count"
},
"query_language": "spl",
"sort_order": 7
},
{
"title": "Best sellers",
"query": "event_kind=order | stats count by sku | sort -count | head 10",
"viz_type": "top_n",
"position_x": 0,
"position_y": 13,
"width": 6,
"height": 5,
"viz_config": {
"label_column": "sku",
"value_column": "count"
},
"query_language": "spl",
"sort_order": 8
},
{
"title": "Out of stock",
"query": "service=magento event_kind=out_of_stock | stats count by sku | sort -count",
"viz_type": "bar",
"position_x": 6,
"position_y": 13,
"width": 6,
"height": 5,
"viz_config": {
"x_column": "sku",
"value_column": "count"
},
"query_language": "spl",
"sort_order": 9
},
{
"title": "Checkout errors",
"query": "service=woocommerce event_kind=checkout_error | sort -timestamp | head 20 | fields timestamp, host, message",
"viz_type": "table",
"position_x": 0,
"position_y": 18,
"width": 6,
"height": 5,
"query_language": "spl",
"sort_order": 10
},
{
"title": "Storefront exceptions",
"query": "event_kind=exception | sort -timestamp | head 20 | fields timestamp, host, service, message",
"viz_type": "table",
"position_x": 6,
"position_y": 18,
"width": 6,
"height": 5,
"query_language": "spl",
"sort_order": 11
}
]
}
hack/demo-seed/dashboards/fleet-health.json
+139
View File
@@ -0,0 +1,139 @@
{
"name": "Fleet health",
"description": "Fifty hosts at a glance: who is loud, who is quiet, and who is running hot",
"default_earliest": "-24h",
"default_latest": "now",
"panels": [
{
"title": "Log events",
"query": "cairnobs.metrics!=true cairnobs.heartbeat!=true | stats count",
"viz_type": "single_stat",
"position_x": 0,
"position_y": 0,
"width": 3,
"height": 3,
"query_language": "spl",
"sort_order": 0
},
{
"title": "Errors",
"query": "severity=ERROR | stats count",
"viz_type": "single_stat",
"position_x": 3,
"position_y": 0,
"width": 3,
"height": 3,
"query_language": "spl",
"sort_order": 1
},
{
"title": "Warnings",
"query": "severity=WARN | stats count",
"viz_type": "single_stat",
"position_x": 6,
"position_y": 0,
"width": 3,
"height": 3,
"query_language": "spl",
"sort_order": 2
},
{
"title": "Fatal",
"query": "severity=FATAL | stats count",
"viz_type": "single_stat",
"position_x": 9,
"position_y": 0,
"width": 3,
"height": 3,
"query_language": "spl",
"sort_order": 3
},
{
"title": "Log volume by host",
"query": "cairnobs.metrics!=true cairnobs.heartbeat!=true | stats count by host | sort -count",
"viz_type": "bar",
"position_x": 0,
"position_y": 3,
"width": 12,
"height": 6,
"viz_config": {
"x_column": "host",
"value_column": "count"
},
"query_language": "spl",
"sort_order": 4
},
{
"title": "Busiest CPU",
"query": "cairnobs.metrics=true | stats avg(cpu_percent) as avg_cpu by host | sort -avg_cpu | head 12",
"viz_type": "top_n",
"position_x": 0,
"position_y": 9,
"width": 6,
"height": 5,
"viz_config": {
"label_column": "host",
"value_column": "avg_cpu"
},
"query_language": "spl",
"sort_order": 5
},
{
"title": "Fullest disks",
"query": "cairnobs.metrics=true | stats max(disk_used_bytes) as used by host | sort -used | head 12",
"viz_type": "top_n",
"position_x": 6,
"position_y": 9,
"width": 6,
"height": 5,
"viz_config": {
"label_column": "host",
"value_column": "used"
},
"query_language": "spl",
"sort_order": 6
},
{
"title": "Severity mix by service",
"query": "cairnobs.metrics!=true | stats count by service, severity",
"viz_type": "heatmap",
"position_x": 0,
"position_y": 14,
"width": 12,
"height": 6,
"viz_config": {
"x_column": "service",
"y_column": "severity",
"value_column": "count"
},
"query_language": "spl",
"sort_order": 7
},
{
"title": "Agents by version",
"query": "cairnobs.heartbeat=true | stats count by agent_version | sort -count",
"viz_type": "bar",
"position_x": 0,
"position_y": 20,
"width": 6,
"height": 4,
"viz_config": {
"x_column": "agent_version",
"value_column": "count"
},
"query_language": "spl",
"sort_order": 8
},
{
"title": "Quietest hosts",
"query": "cairnobs.metrics!=true cairnobs.heartbeat!=true | stats count by host | sort count | head 10",
"viz_type": "table",
"position_x": 6,
"position_y": 20,
"width": 6,
"height": 4,
"query_language": "spl",
"sort_order": 9
}
]
}
hack/demo-seed/dashboards/golden-signals.json
+133
View File
@@ -0,0 +1,133 @@
{
"name": "Golden signals",
"description": "Traffic, errors, latency and saturation across everything that serves a request",
"default_earliest": "-6h",
"default_latest": "now",
"panels": [
{
"title": "Requests served",
"query": "| where status>=100 | stats count",
"viz_type": "single_stat",
"position_x": 0,
"position_y": 0,
"width": 3,
"height": 3,
"query_language": "spl",
"sort_order": 0
},
{
"title": "5xx responses",
"query": "| where status>=500 | stats count",
"viz_type": "single_stat",
"position_x": 3,
"position_y": 0,
"width": 3,
"height": 3,
"query_language": "spl",
"sort_order": 1
},
{
"title": "API latency (avg ms)",
"query": "service=api | stats avg(duration_ms) as avg_ms",
"viz_type": "single_stat",
"position_x": 6,
"position_y": 0,
"width": 3,
"height": 3,
"query_language": "spl",
"sort_order": 2
},
{
"title": "Slow Postgres statements",
"query": "service=postgres | where duration_ms>1000 | stats count",
"viz_type": "single_stat",
"position_x": 9,
"position_y": 0,
"width": 3,
"height": 3,
"query_language": "spl",
"sort_order": 3
},
{
"title": "Traffic by service and host",
"query": "| where status>=100 | stats count by service, host",
"viz_type": "bar",
"position_x": 0,
"position_y": 3,
"width": 12,
"height": 5,
"viz_config": {
"x_column": "service",
"value_column": "count",
"series_column": "host",
"stacked": "true"
},
"query_language": "spl",
"sort_order": 4
},
{
"title": "Where the errors are",
"query": "severity=ERROR | stats count by service, host | sort -count | head 20",
"viz_type": "table",
"position_x": 0,
"position_y": 8,
"width": 6,
"height": 5,
"query_language": "spl",
"sort_order": 5
},
{
"title": "API latency by route",
"query": "service=api | stats avg(duration_ms) as avg_ms by route | sort -avg_ms | head 12",
"viz_type": "bar",
"position_x": 6,
"position_y": 8,
"width": 6,
"height": 5,
"viz_config": {
"x_column": "route",
"value_column": "avg_ms"
},
"query_language": "spl",
"sort_order": 6
},
{
"title": "Status codes by service",
"query": "| where status>=100 | stats count by service, status",
"viz_type": "heatmap",
"position_x": 0,
"position_y": 13,
"width": 12,
"height": 5,
"viz_config": {
"x_column": "service",
"y_column": "status",
"value_column": "count"
},
"query_language": "spl",
"sort_order": 7
},
{
"title": "Slow operations, wherever they are",
"query": "| where duration_ms>1000 | stats count by service, host | sort -count | head 15",
"viz_type": "table",
"position_x": 0,
"position_y": 18,
"width": 6,
"height": 5,
"query_language": "spl",
"sort_order": 8
},
{
"title": "Saturation by host",
"query": "cairnobs.metrics=true | stats avg(cpu_percent) as avg_cpu, max(mem_used_bytes) as peak_mem by host | sort -avg_cpu | head 15",
"viz_type": "table",
"position_x": 6,
"position_y": 18,
"width": 6,
"height": 5,
"query_language": "spl",
"sort_order": 9
}
]
}
hack/demo-seed/dashboards/payments.json
+141
View File
@@ -0,0 +1,141 @@
{
"name": "Payments",
"description": "Authorisations, declines and the gateways behind them -- where money stops moving and why",
"default_earliest": "-24h",
"default_latest": "now",
"panels": [
{
"title": "Authorisations",
"query": "event_kind=authorization | stats count",
"viz_type": "single_stat",
"position_x": 0,
"position_y": 0,
"width": 3,
"height": 3,
"query_language": "spl",
"sort_order": 0
},
{
"title": "Declines",
"query": "auth_result=declined | stats count",
"viz_type": "single_stat",
"position_x": 3,
"position_y": 0,
"width": 3,
"height": 3,
"query_language": "spl",
"sort_order": 1
},
{
"title": "Refunds",
"query": "event_kind=refund | stats count",
"viz_type": "single_stat",
"position_x": 6,
"position_y": 0,
"width": 3,
"height": 3,
"query_language": "spl",
"sort_order": 2
},
{
"title": "Chargebacks",
"query": "event_kind=chargeback | stats count",
"viz_type": "single_stat",
"position_x": 9,
"position_y": 0,
"width": 3,
"height": 3,
"query_language": "spl",
"sort_order": 3
},
{
"title": "Approved and declined by gateway",
"query": "event_kind=authorization | stats count by gateway, auth_result",
"viz_type": "bar",
"position_x": 0,
"position_y": 3,
"width": 6,
"height": 5,
"viz_config": {
"x_column": "gateway",
"value_column": "count",
"series_column": "auth_result",
"stacked": "true"
},
"query_language": "spl",
"sort_order": 4
},
{
"title": "Why cards are declined",
"query": "auth_result=declined | stats count by decline_reason | sort -count",
"viz_type": "bar",
"position_x": 6,
"position_y": 3,
"width": 6,
"height": 5,
"viz_config": {
"x_column": "decline_reason",
"value_column": "count"
},
"query_language": "spl",
"sort_order": 5
},
{
"title": "Declined value",
"query": "auth_result=declined | stats sum(amount) as lost by gateway | sort -lost",
"viz_type": "top_n",
"position_x": 0,
"position_y": 8,
"width": 6,
"height": 5,
"viz_config": {
"label_column": "gateway",
"value_column": "lost"
},
"query_language": "spl",
"sort_order": 6
},
{
"title": "Gateway latency",
"query": "event_kind=authorization | stats avg(duration_ms) as avg_ms by gateway | sort -avg_ms",
"viz_type": "bar",
"position_x": 6,
"position_y": 8,
"width": 6,
"height": 5,
"viz_config": {
"x_column": "gateway",
"value_column": "avg_ms"
},
"query_language": "spl",
"sort_order": 7
},
{
"title": "Decline reasons by gateway",
"query": "auth_result=declined | stats count by gateway, decline_reason",
"viz_type": "heatmap",
"position_x": 0,
"position_y": 13,
"width": 12,
"height": 5,
"viz_config": {
"x_column": "gateway",
"y_column": "decline_reason",
"value_column": "count"
},
"query_language": "spl",
"sort_order": 8
},
{
"title": "Chargebacks and refunds",
"query": "service=payments | where event_kind=chargeback | sort -timestamp | head 20 | fields timestamp, gateway, amount, message",
"viz_type": "table",
"position_x": 0,
"position_y": 18,
"width": 12,
"height": 5,
"query_language": "spl",
"sort_order": 9
}
]
}
hack/demo-seed/dashboards/security-posture.json
+145
View File
@@ -0,0 +1,145 @@
{
"name": "Security posture",
"description": "Authentication, authorisation and the outbound edge -- every place a credential or a request gets refused",
"default_earliest": "-24h",
"default_latest": "now",
"panels": [
{
"title": "Failed SSH",
"query": "service=system auth_result=failed | stats count",
"viz_type": "single_stat",
"position_x": 0,
"position_y": 0,
"width": 3,
"height": 3,
"query_language": "spl",
"sort_order": 0
},
{
"title": "Failed Windows logons",
"query": "service=eventlog winevt.event_id=4625 | stats count",
"viz_type": "single_stat",
"position_x": 3,
"position_y": 0,
"width": 3,
"height": 3,
"query_language": "spl",
"sort_order": 1
},
{
"title": "Failed SQL logins",
"query": "service=mssql event_kind=login_failed | stats count",
"viz_type": "single_stat",
"position_x": 6,
"position_y": 0,
"width": 3,
"height": 3,
"query_language": "spl",
"sort_order": 2
},
{
"title": "Denied share access",
"query": "service=smb event_kind=share_denied | stats count",
"viz_type": "single_stat",
"position_x": 9,
"position_y": 0,
"width": 3,
"height": 3,
"query_language": "spl",
"sort_order": 3
},
{
"title": "Source addresses probing SSH",
"query": "service=system auth_result=failed | stats count by remote_addr | sort -count | head 12",
"viz_type": "top_n",
"position_x": 0,
"position_y": 3,
"width": 6,
"height": 5,
"viz_config": {
"label_column": "remote_addr",
"value_column": "count"
},
"query_language": "spl",
"sort_order": 4
},
{
"title": "Accounts targeted on Windows",
"query": "service=eventlog winevt.event_id=4625 | stats count by winevt.target_user | sort -count",
"viz_type": "bar",
"position_x": 6,
"position_y": 3,
"width": 6,
"height": 5,
"viz_config": {
"x_column": "winevt.target_user",
"value_column": "count"
},
"query_language": "spl",
"sort_order": 5
},
{
"title": "Blocked outbound destinations",
"query": "service=squid status=403 | stats count by dest_host | sort -count",
"viz_type": "bar",
"position_x": 0,
"position_y": 8,
"width": 6,
"height": 5,
"viz_config": {
"x_column": "dest_host",
"value_column": "count"
},
"query_language": "spl",
"sort_order": 6
},
{
"title": "NXDOMAIN by client",
"query": "service=bind dns_rcode=NXDOMAIN | stats count by remote_addr | sort -count | head 12",
"viz_type": "top_n",
"position_x": 6,
"position_y": 8,
"width": 6,
"height": 5,
"viz_config": {
"label_column": "remote_addr",
"value_column": "count"
},
"query_language": "spl",
"sort_order": 7
},
{
"title": "Failed LDAP binds",
"query": "service=openldap event_kind=bind_failed | sort -timestamp | head 20 | fields timestamp, host, bind_dn, remote_addr, message",
"viz_type": "table",
"position_x": 0,
"position_y": 13,
"width": 6,
"height": 5,
"query_language": "spl",
"sort_order": 8
},
{
"title": "Vault authentication failures",
"query": "service=vault event_kind=auth_failed | sort -timestamp | head 20 | fields timestamp, host, vault_path, remote_addr, message",
"viz_type": "table",
"position_x": 6,
"position_y": 13,
"width": 6,
"height": 5,
"query_language": "spl",
"sort_order": 9
},
{
"title": "Account lockouts",
"query": "service=eventlog winevt.event_id=4740 | sort -timestamp | head 20 | fields timestamp, host, winevt.target_user, message",
"viz_type": "table",
"position_x": 0,
"position_y": 18,
"width": 12,
"height": 5,
"query_language": "spl",
"sort_order": 10
}
]
}