From acbf6f7fec105b1690ccb5b5e89d1cb5076962a0 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Wed, 30 Sep 2026 11:40:21 -0700 Subject: [PATCH] Dashboard: diff histogram totals per node Each node stores its histograms as running totals since it started. deltaHistograms lined up every node's samples by time and diffed them in one list, so in a cluster it subtracted one node's total from another's. The timing averages came out wrong: on production the delivery attempt time read 14.7 s over the last hour ("1 stage running slow") when the nodes' own figures give 0.7 s. Samples now carry nodeId (inbuxa-server, x:Metric), and totals are diffed per metric and node. Without nodeId (older servers, single node) it works as before. --- src/features/dashboard/helpers.test.ts | 62 +++++++++++++++++++++++++ src/features/dashboard/helpers.ts | 13 ++++-- src/features/dashboard/types/metrics.ts | 10 ++-- 3 files changed, 79 insertions(+), 6 deletions(-) create mode 100644 src/features/dashboard/helpers.test.ts diff --git a/src/features/dashboard/helpers.test.ts b/src/features/dashboard/helpers.test.ts new file mode 100644 index 0000000..0bde2a6 --- /dev/null +++ b/src/features/dashboard/helpers.test.ts @@ -0,0 +1,62 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs + * + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { describe, expect, it } from 'vitest'; +import type { Metric } from './types/metrics'; +import { deltaHistograms } from './helpers'; + +const total = (timestamp: string, count: number, sum: number, nodeId?: number): Metric => ({ + '@type': 'Histogram', + metric: 'delivery.attempt-time', + count, + sum, + timestamp, + nodeId, +}); + +const sums = (samples: Metric[]) => + samples.reduce( + (acc, m) => (m['@type'] === 'Histogram' ? { count: acc.count + m.count, sum: acc.sum + m.sum } : acc), + { count: 0, sum: 0 }, + ); + +describe('deltaHistograms', () => { + it('diffs one node’s running totals', () => { + const out = deltaHistograms([ + total('2026-09-30T10:00:00Z', 10, 1000), + total('2026-09-30T11:00:00Z', 12, 1400), + total('2026-09-30T12:00:00Z', 15, 2000), + ]); + expect(sums(out)).toEqual({ count: 5, sum: 1000 }); + }); + + it('diffs each node against itself, never against another node', () => { + // A busy fast node and a quiet node with one slow attempt, interleaved + const out = deltaHistograms([ + total('2026-09-30T10:00:00Z', 100, 20_000, 0), + total('2026-09-30T10:00:01Z', 4, 300_000, 1), + total('2026-09-30T11:00:00Z', 110, 22_000, 0), + total('2026-09-30T11:00:01Z', 4, 300_000, 1), + total('2026-09-30T12:00:00Z', 120, 24_000, 0), + total('2026-09-30T12:00:01Z', 5, 301_000, 1), + ]); + expect(sums(out)).toEqual({ count: 21, sum: 5_000 }); + }); + + it('skips a restart, where a node’s totals start again from zero', () => { + const out = deltaHistograms([ + total('2026-09-30T10:00:00Z', 50, 5000, 2), + total('2026-09-30T11:00:00Z', 3, 300, 2), + total('2026-09-30T12:00:00Z', 5, 700, 2), + ]); + expect(sums(out)).toEqual({ count: 2, sum: 400 }); + }); + + it('passes counters and gauges through untouched', () => { + const counter: Metric = { '@type': 'Counter', metric: 'auth.success', count: 3, nodeId: 1 }; + expect(deltaHistograms([counter])).toEqual([counter]); + }); +}); diff --git a/src/features/dashboard/helpers.ts b/src/features/dashboard/helpers.ts index 7e45560..a410c9a 100644 --- a/src/features/dashboard/helpers.ts +++ b/src/features/dashboard/helpers.ts @@ -1,7 +1,10 @@ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * SPDX-FileCopyrightText: 2026 Coffey Labs * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + * + * Modified by Coffey Labs in 2026 for INBUXA. */ import type { Metric, MetricId, Period } from './types/metrics'; @@ -15,19 +18,23 @@ export function metricToScalar(m: Metric): number { return m.count; } +// INBUXA: histograms are running totals per node, so they're diffed per +// node; in a cluster, diffing across nodes subtracts one node's total from +// another's. export function deltaHistograms(samples: Metric[]): Metric[] { const nonHistograms: Metric[] = []; - const byMetric = new Map(); + const byMetric = new Map(); for (const m of samples) { if (m['@type'] !== 'Histogram') { nonHistograms.push(m); continue; } - let list = byMetric.get(m.metric); + const key = `${m.metric}|${m.nodeId ?? ''}`; + let list = byMetric.get(key); if (!list) { list = []; - byMetric.set(m.metric, list); + byMetric.set(key, list); } list.push(m); } diff --git a/src/features/dashboard/types/metrics.ts b/src/features/dashboard/types/metrics.ts index 1c6600c..0495a80 100644 --- a/src/features/dashboard/types/metrics.ts +++ b/src/features/dashboard/types/metrics.ts @@ -1,15 +1,19 @@ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * SPDX-FileCopyrightText: 2026 Coffey Labs * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + * + * Modified by Coffey Labs in 2026 for INBUXA. */ export type MetricId = string; +// INBUXA: nodeId is the node that wrote the sample (servers from 2026.9.30 on) export type Metric = - | { '@type': 'Counter'; metric: MetricId; count: number; timestamp?: string } - | { '@type': 'Gauge'; metric: MetricId; count: number; timestamp?: string } - | { '@type': 'Histogram'; metric: MetricId; count: number; sum: number; timestamp?: string }; + | { '@type': 'Counter'; metric: MetricId; count: number; timestamp?: string; nodeId?: number } + | { '@type': 'Gauge'; metric: MetricId; count: number; timestamp?: string; nodeId?: number } + | { '@type': 'Histogram'; metric: MetricId; count: number; sum: number; timestamp?: string; nodeId?: number }; export type Period = { kind: 'preset'; preset: PresetKey } | { kind: 'custom'; from: Date; to: Date }; -- 2.54.0