Dashboard: read gauges per node #76

Merged
jcoffey-dev merged 2 commits from fix/gauges-per-node into main 2026-10-01 02:01:46 +00:00
5 changed files with 119 additions and 17 deletions

No files matched your search

+8 -5
View File
@@ -10,7 +10,7 @@
* from the components so every figure can be tested on its own.
*/
import type { Metric } from './types/metrics';
import { bucketize, metricToScalar } from './helpers';
import { bucketize, gaugeReading, metricToScalar } from './helpers';
/** Messages the server took in from outside. */
export const RECEIVED = ['queue.message-queued'];
@@ -107,9 +107,12 @@ export function mean(samples: Metric[], ids: string[]): number | null {
/** Counts per time slice across the period, oldest first. */
export function series(samples: Metric[], ids: string[], from: Date, to: Date, buckets: number): number[] {
return bucketize(samples, from, to, buckets).map((b) =>
b.filter((m) => ids.includes(m.metric)).reduce((s, m) => s + metricToScalar(m), 0),
);
return bucketize(samples, from, to, buckets).map((b) => {
const mine = b.filter((m) => ids.includes(m.metric));
// A gauge's slice is one reading across the nodes, not the sum of every tick
if (mine.length > 0 && mine.every((m) => m['@type'] === 'Gauge')) return gaugeReading(mine, ids) ?? 0;
return mine.reduce((s, m) => s + metricToScalar(m), 0);
});
}
export interface Load {
@@ -180,7 +183,7 @@ export function latest(samples: Metric[], ids: string[]): number | null {
export function summarize(samples: Metric[], ids: string[], timing: boolean): number | null {
if (timing) return mean(samples, ids);
const mine = samples.filter((s) => ids.includes(s.metric));
if (mine.length > 0 && mine.every((s) => s['@type'] === 'Gauge')) return latest(mine, ids);
if (mine.length > 0 && mine.every((s) => s['@type'] === 'Gauge')) return gaugeReading(mine, ids);
return total(samples, ids);
}
@@ -8,7 +8,7 @@ import type { ReactNode } from 'react';
import { useTranslation } from 'react-i18next';
import type { Metric, Period } from '../types/metrics';
import type { ServerFacts } from '../serverFacts';
import { bucketTimestamps, formatValue, getBucketCount } from '../helpers';
import { bucketTimestamps, formatValue, gaugeReading, getBucketCount } from '../helpers';
import {
DECK_METRICS,
AUTH_FAILED,
@@ -18,7 +18,6 @@ import {
RECEIVED,
SENT,
SPAM,
latest,
load,
mean,
queueHealth,
@@ -149,11 +148,12 @@ export function TrendDeck({ id, samples, window, period, periodName, live, liveV
/>
);
// Memory now against its highest in the period: how close to the peak the node is running.
// Memory now against its highest in the period, both summed over the nodes:
// the live feed is one node's, so the reading comes from the stored history.
const memoryGauge = () => {
const readings = samples.filter((m) => m.metric === 'server.memory').map((m) => m.count);
const now = liveValues.get('server.memory') ?? latest(samples, ['server.memory']);
const peak = Math.max(0, now ?? 0, ...readings);
const MEMORY = ['server.memory'];
const now = gaugeReading(samples, MEMORY);
const peak = Math.max(0, now ?? 0, ...series(samples, MEMORY, from, to, buckets));
return (
<Gauge
key="memory"
@@ -163,7 +163,7 @@ export function TrendDeck({ id, samples, window, period, periodName, live, liveV
label={t('cc.g.memory', 'Memory')}
detail={
peak
? t('cc.g.memoryDetail', 'this node · peak {{peak}}', { peak: formatValue(peak, 'bytes') })
? t('cc.g.memoryDetail', 'all nodes · peak {{peak}}', { peak: formatValue(peak, 'bytes') })
: t('cc.g.quiet', 'Nothing recorded yet')
}
/>
+60 -1
View File
@@ -6,7 +6,7 @@
import { describe, expect, it } from 'vitest';
import type { Metric } from './types/metrics';
import { deltaHistograms } from './helpers';
import { deltaHistograms, gaugeReading, seriesBucketValue } from './helpers';
const total = (timestamp: string, count: number, sum: number, nodeId?: number): Metric => ({
'@type': 'Histogram',
@@ -60,3 +60,62 @@ describe('deltaHistograms', () => {
expect(deltaHistograms([counter])).toEqual([counter]);
});
});
const reading = (metric: string, timestamp: string, count: number, nodeId?: number): Metric => ({
'@type': 'Gauge',
metric,
count,
timestamp,
nodeId,
});
describe('gaugeReading', () => {
it('adds up each node’s latest reading of a per-node gauge', () => {
const v = gaugeReading(
[
reading('server.memory', '2026-09-30T10:00:00Z', 100, 0),
reading('server.memory', '2026-09-30T11:00:00Z', 180, 0),
reading('server.memory', '2026-09-30T11:00:00Z', 200, 1),
reading('server.memory', '2026-09-30T11:00:01Z', 220, 2),
],
['server.memory'],
);
expect(v).toBe(600);
});
it('takes the real cluster-wide count, not a drifted or zero copy', () => {
const samples = [
reading('queue.count', '2026-09-30T11:00:00Z', 8, 2),
reading('queue.count', '2026-09-30T11:00:01Z', 2 ** 64 - 20, 0),
reading('queue.count', '2026-09-30T11:00:02Z', 2 ** 64 - 3, 1),
reading('user.count', '2026-09-30T11:00:00Z', 7, 2),
reading('user.count', '2026-09-30T11:00:02Z', 0, 1),
];
expect(gaugeReading(samples, ['queue.count'])).toBe(8);
expect(gaugeReading(samples, ['user.count'])).toBe(7);
});
it('is null with no reading, or only an unbelievable one', () => {
expect(gaugeReading([], ['server.memory'])).toBeNull();
expect(gaugeReading([reading('queue.count', '2026-09-30T11:00:00Z', 2 ** 64 - 3, 0)], ['queue.count'])).toBeNull();
});
it('reads a single-node server (no nodeId) as before', () => {
const v = gaugeReading(
[reading('server.memory', '2026-09-30T10:00:00Z', 100), reading('server.memory', '2026-09-30T11:00:00Z', 150)],
['server.memory'],
);
expect(v).toBe(150);
});
});
describe('seriesBucketValue', () => {
it('reads a gauge slice across the nodes instead of adding up its ticks', () => {
const slice = [
reading('server.memory', '2026-09-30T10:00:00Z', 100, 0),
reading('server.memory', '2026-09-30T10:01:00Z', 110, 0),
reading('server.memory', '2026-09-30T10:00:30Z', 50, 1),
];
expect(seriesBucketValue({ label: 'Memory', metrics: ['server.memory'] }, slice)).toBe(160);
});
});
+43 -3
View File
@@ -99,9 +99,49 @@ export function cardValue(card: Card, samples: Metric[]): number {
}
export function seriesBucketValue(series: Series, bucketSamples: Metric[]): number | null {
const values = bucketSamples.filter((m) => series.metrics.includes(m.metric)).map(metricToScalar);
if (values.length === 0) return null;
return aggregate(values, series.aggregate);
const mine = bucketSamples.filter((m) => series.metrics.includes(m.metric));
if (mine.length === 0) return null;
// INBUXA: a gauge's slice is one reading across the nodes, not every tick's readings combined
if (mine.every((m) => m['@type'] === 'Gauge')) return gaugeReading(mine, series.metrics);
return aggregate(mine.map(metricToScalar), series.aggregate);
}
// INBUXA: gauges counting the whole cluster. Only the node that computes
// them has a true reading; history from before inbuxa-server 2026.9.30.2
// also holds the other nodes' copies, with the queue drifted below zero
// (stored unsigned, so a 20-digit number) and the accounts and domains at 0.
export const CLUSTER_GAUGES = new Set<MetricId>(['queue.count', 'user.count', 'domain.count']);
/**
* The gauges' reading across the cluster: each node's latest, added up for
* per-node gauges (memory, open connections), and for cluster-wide counts the
* largest believable one. Null when there's no reading.
*/
export function gaugeReading(samples: Metric[], ids: MetricId[]): number | null {
const last = new Map<string, { metric: MetricId; at: number; value: number }>();
for (const m of samples) {
if (m['@type'] !== 'Gauge' || !ids.includes(m.metric)) continue;
const at = m.timestamp ? Date.parse(m.timestamp) : 0;
const key = `${m.metric}|${m.nodeId ?? ''}`;
const prev = last.get(key);
if (!prev || at >= prev.at) last.set(key, { metric: m.metric, at, value: m.count });
}
let sum = 0;
let found = false;
const cluster = new Map<MetricId, number>();
for (const { metric, value } of last.values()) {
if (!CLUSTER_GAUGES.has(metric)) {
sum += value;
found = true;
} else if (value <= Number.MAX_SAFE_INTEGER) {
cluster.set(metric, Math.max(cluster.get(metric) ?? 0, value));
}
}
for (const value of cluster.values()) {
sum += value;
found = true;
}
return found ? sum : null;
}
export function bucketize(samples: Metric[], from: Date, to: Date, bucketCount: number): Metric[][] {
+1 -1
View File
@@ -1841,7 +1841,7 @@
"zoneDetail": "fine under {{good}} · slow over {{bad}}",
"quiet": "Nothing recorded yet",
"memory": "Memory",
"memoryDetail": "this node · peak {{peak}}",
"memoryDetail": "all nodes · peak {{peak}}",
"connLoad": "Connection load",
"smtpSplit": "SMTP sessions",
"smtpIn": "{{n}} in",