Cluster Health: readable on Kubernetes, no pile-up after rollouts (K-A3) #258
No files matched your search
@@ -5,7 +5,7 @@
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { minutesSilent, setOf, summarizeNodes, unboundOf, type ClusterNodeRow } from './clusterHealth';
|
||||
import { minutesSilent, podName, setOf, summarizeNodes, unboundOf, type ClusterNodeRow } from './clusterHealth';
|
||||
|
||||
const node = (hostname: string, status: ClusterNodeRow['status'], lastRenewal = '2026-09-25T10:00:00Z') => ({
|
||||
hostname,
|
||||
@@ -27,7 +27,7 @@ describe('summarizeNodes', () => {
|
||||
node('mx2.example.org', 'active'),
|
||||
node('mx3.example.org', 'active'),
|
||||
]),
|
||||
).toEqual({ healthy: 3, unhealthy: 0, silent: [] });
|
||||
).toEqual({ healthy: 3, unhealthy: 0, silent: [], gone: [] });
|
||||
});
|
||||
|
||||
it('names the silent nodes, longest quiet first', () => {
|
||||
@@ -53,6 +53,63 @@ describe('summarizeNodes', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('pods (K-A3)', () => {
|
||||
it('reads Kubernetes pod names, and nothing else', () => {
|
||||
expect(podName('inbuxa-mta-674c7dffc5-c9bp7')).toEqual({
|
||||
group: 'inbuxa-mta',
|
||||
pod: '674c7dffc5-c9bp7',
|
||||
replicated: true,
|
||||
});
|
||||
expect(podName('inbuxa-upgrade-fcxxt')).toEqual({ group: 'inbuxa-upgrade', pod: 'fcxxt', replicated: false });
|
||||
expect(podName('mx2.example.org')).toBeNull();
|
||||
expect(podName('mail')).toBeNull();
|
||||
expect(podName('inbuxa-0')).toBeNull(); // a StatefulSet's pod keeps its name
|
||||
expect(podName('mail-relay')).toBeNull(); // vowels: a word, not a generated suffix
|
||||
});
|
||||
|
||||
// The lab after a few rollouts: five pods renewing, four replaced, and the
|
||||
// upgrade hook's pod that ran once.
|
||||
const lab = [
|
||||
node('inbuxa-core-786fbfd875-wftz8', 'active', '2026-11-20T07:24:47Z'),
|
||||
node('inbuxa-front-75d4794556-9nv8k', 'active', '2026-11-20T07:24:45Z'),
|
||||
node('inbuxa-front-75d4794556-x7tjq', 'active', '2026-11-20T07:24:49Z'),
|
||||
node('inbuxa-mta-674c7dffc5-c9bp7', 'active', '2026-11-20T07:24:45Z'),
|
||||
node('inbuxa-mta-674c7dffc5-jj6br', 'active', '2026-11-20T07:24:47Z'),
|
||||
node('inbuxa-front-75d4794556-4m47f', 'stale', '2026-11-20T06:49:40Z'),
|
||||
node('inbuxa-front-75d4794556-v2zxq', 'stale', '2026-11-20T06:50:03Z'),
|
||||
node('inbuxa-mta-674c7dffc5-5f6v9', 'stale', '2026-11-20T06:49:49Z'),
|
||||
node('inbuxa-mta-674c7dffc5-9g5nx', 'stale', '2026-11-20T06:50:03Z'),
|
||||
node('inbuxa-upgrade-fcxxt', 'stale', '2026-11-20T07:19:22Z'),
|
||||
];
|
||||
|
||||
it('does not count pods a rollout replaced, or a finished Job', () => {
|
||||
const health = summarizeNodes(lab);
|
||||
expect(health).toMatchObject({ healthy: 5, unhealthy: 0, silent: [] });
|
||||
expect(health?.gone.map((n) => n.hostname)).toEqual([
|
||||
'inbuxa-upgrade-fcxxt',
|
||||
'inbuxa-front-75d4794556-v2zxq',
|
||||
'inbuxa-mta-674c7dffc5-9g5nx',
|
||||
'inbuxa-mta-674c7dffc5-5f6v9',
|
||||
'inbuxa-front-75d4794556-4m47f',
|
||||
]);
|
||||
});
|
||||
|
||||
it('counts a role with no pod renewing as down, once', () => {
|
||||
const health = summarizeNodes(
|
||||
lab.filter((n) => !n.hostname.startsWith('inbuxa-mta-674c7dffc5-') || n.status !== 'active'),
|
||||
);
|
||||
expect(health).toMatchObject({ healthy: 3, unhealthy: 1 });
|
||||
expect(health?.silent.map((n) => n.hostname)).toEqual(['inbuxa-mta-674c7dffc5-9g5nx']);
|
||||
expect(health?.gone.map((n) => n.hostname)).toContain('inbuxa-mta-674c7dffc5-5f6v9');
|
||||
});
|
||||
|
||||
it('still counts a silent server that is not a pod', () => {
|
||||
const health = summarizeNodes([...lab, node('mx2.example.org', 'stale', '2026-11-20T07:00:00Z')]);
|
||||
expect(health).toMatchObject({ healthy: 5, unhealthy: 1 });
|
||||
expect(health?.silent.map((n) => n.hostname)).toEqual(['mx2.example.org']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('minutesSilent', () => {
|
||||
it('counts whole minutes since the last renewal', () => {
|
||||
const now = Date.parse('2026-09-25T10:07:30Z');
|
||||
@@ -77,7 +134,9 @@ describe('unboundOf (inbuxa-server#485)', () => {
|
||||
describe('setOf', () => {
|
||||
it('reads a registry set as the server sends it', () => {
|
||||
// As a live server answered for a listener held by another process
|
||||
const wire = { 'imaps [::]:12993: Failed to bind to [::]:12993: Address already in use (os error 98); retrying': true };
|
||||
const wire = {
|
||||
'imaps [::]:12993: Failed to bind to [::]:12993: Address already in use (os error 98); retrying': true,
|
||||
};
|
||||
expect(setOf(wire)).toEqual([
|
||||
'imaps [::]:12993: Failed to bind to [::]:12993: Address already in use (os error 98); retrying',
|
||||
]);
|
||||
|
||||
@@ -46,21 +46,80 @@ export function unboundOf(nodes: ClusterNodeRow[]): UnboundNode[] {
|
||||
.map(({ hostname, unboundListeners }) => ({ hostname, listeners: unboundListeners }));
|
||||
}
|
||||
|
||||
/*
|
||||
* inbuxa: K-A3. On Kubernetes a node's hostname is its pod's name, and the
|
||||
* name says what made the pod: a Deployment's pods are named
|
||||
* `<deployment>-<template hash>-<5>`, a Job's (and a DaemonSet's)
|
||||
* `<job>-<5>`, the generated parts drawn from an alphabet with no vowels and
|
||||
* no 0, 1 or 3. A pod's name is never used again: one that stopped renewing
|
||||
* isn't coming back, its replacement leases under a new name. So a silent
|
||||
* lease with a pod's name is a pod that's gone (a rollout replaced it, or a
|
||||
* Job finished), not a node that's down, and it's left out of the count,
|
||||
* which would otherwise be half red for a day after every rollout. Only
|
||||
* when no pod of a Deployment is renewing at all is the role down, and then
|
||||
* it counts once.
|
||||
*/
|
||||
const POD_CHARS = '[bcdfghjklmnpqrstvwxz2456789]';
|
||||
const DEPLOYMENT_POD = new RegExp(`^([a-z0-9][a-z0-9-]*?)-(${POD_CHARS}{6,10})-(${POD_CHARS}{5})$`);
|
||||
const OTHER_POD = new RegExp(`^([a-z0-9][a-z0-9-]*?)-(${POD_CHARS}{5})$`);
|
||||
|
||||
export interface PodName {
|
||||
/** What made the pod: the Deployment, Job or DaemonSet's name. */
|
||||
group: string;
|
||||
/** The generated part that tells this pod from its siblings. */
|
||||
pod: string;
|
||||
/** A Deployment's pod, which is replaced when it goes. */
|
||||
replicated: boolean;
|
||||
}
|
||||
|
||||
/** The parts of a Kubernetes pod name, or null for any other hostname. */
|
||||
export function podName(hostname: string): PodName | null {
|
||||
const d = DEPLOYMENT_POD.exec(hostname);
|
||||
if (d) return { group: d[1], pod: `${d[2]}-${d[3]}`, replicated: true };
|
||||
const o = OTHER_POD.exec(hostname);
|
||||
return o ? { group: o[1], pod: o[2], replicated: false } : null;
|
||||
}
|
||||
|
||||
export interface ClusterHealth {
|
||||
healthy: number;
|
||||
unhealthy: number;
|
||||
/** Nodes that stopped renewing, longest silent first. */
|
||||
silent: { hostname: string; lastRenewal: string }[];
|
||||
/** Pods that stopped renewing and won't be back (see podName), latest first. */
|
||||
gone: { hostname: string; lastRenewal: string }[];
|
||||
}
|
||||
|
||||
export function summarizeNodes(nodes: ClusterNodeRow[]): ClusterHealth | null {
|
||||
const members = nodes.filter((n) => n.status !== 'inactive');
|
||||
if (members.length < 2) return null;
|
||||
const silent = members
|
||||
.filter((n) => n.status !== 'active')
|
||||
const quiet = members.filter((n) => n.status !== 'active');
|
||||
const renewing = new Set(
|
||||
members
|
||||
.filter((n) => n.status === 'active')
|
||||
.map((n) => podName(n.hostname)?.group)
|
||||
.filter(Boolean),
|
||||
);
|
||||
// A Deployment with no pod renewing: its latest pod stands for it
|
||||
const down = new Map<string, ClusterNodeRow>();
|
||||
for (const n of quiet) {
|
||||
const pod = podName(n.hostname);
|
||||
if (!pod?.replicated || renewing.has(pod.group)) continue;
|
||||
const seen = down.get(pod.group);
|
||||
if (!seen || seen.lastRenewal < n.lastRenewal) down.set(pod.group, n);
|
||||
}
|
||||
const standsFor = new Set(down.values());
|
||||
const isGone = (n: ClusterNodeRow) => podName(n.hostname) !== null && !standsFor.has(n);
|
||||
const pick = ({ hostname, lastRenewal }: ClusterNodeRow) => ({ hostname, lastRenewal });
|
||||
const silent = quiet
|
||||
.filter((n) => !isGone(n))
|
||||
.sort((a, b) => a.lastRenewal.localeCompare(b.lastRenewal))
|
||||
.map(({ hostname, lastRenewal }) => ({ hostname, lastRenewal }));
|
||||
return { healthy: members.length - silent.length, unhealthy: silent.length, silent };
|
||||
.map(pick);
|
||||
const gone = quiet
|
||||
.filter(isGone)
|
||||
.sort((a, b) => b.lastRenewal.localeCompare(a.lastRenewal))
|
||||
.map(pick);
|
||||
const healthy = members.length - quiet.length;
|
||||
return { healthy, unhealthy: silent.length, silent, gone };
|
||||
}
|
||||
|
||||
/** Whole minutes since a node last renewed its lease, or null if the time can't be read. */
|
||||
|
||||
@@ -27,7 +27,7 @@ import {
|
||||
series,
|
||||
total,
|
||||
} from '../commandCenter';
|
||||
import type { ClusterNodeRow } from '../clusterHealth';
|
||||
import { podName, type ClusterNodeRow } from '../clusterHealth';
|
||||
import { useDashboardShared } from '../dashboardContext';
|
||||
import { usePageHistory } from '../usePageHistory';
|
||||
import type { Tone } from '../tones';
|
||||
@@ -130,8 +130,22 @@ export function ClusterView() {
|
||||
);
|
||||
|
||||
const { nodes, health } = cluster;
|
||||
const members = nodes.filter((n) => n.status !== 'inactive');
|
||||
// inbuxa: K-A3, pods that are gone (see podName) get a line, not a card
|
||||
const gone = new Set(health.gone.map((g) => g.hostname));
|
||||
const members = nodes.filter((n) => n.status !== 'inactive' && !gone.has(n.hostname));
|
||||
const retired = nodes.filter((n) => n.status === 'inactive');
|
||||
// Cards grouped by what made the pod; servers that aren't pods come first
|
||||
const groups = new Map<string, ClusterNodeRow[]>();
|
||||
for (const node of [...members].sort((a, b) => a.hostname.localeCompare(b.hostname))) {
|
||||
const key = podName(node.hostname)?.group ?? '';
|
||||
groups.set(key, [...(groups.get(key) ?? []), node]);
|
||||
}
|
||||
const grouped = [...groups.entries()].sort(([a], [b]) => (a === '' ? -1 : b === '' ? 1 : a.localeCompare(b)));
|
||||
const goneByGroup = new Map<string, number>();
|
||||
for (const g of health.gone) {
|
||||
const key = podName(g.hostname)?.group ?? g.hostname;
|
||||
goneByGroup.set(key, (goneByGroup.get(key) ?? 0) + 1);
|
||||
}
|
||||
const all = health.healthy + health.unhealthy;
|
||||
const activeAges = members
|
||||
.filter((n) => n.status === 'active')
|
||||
@@ -242,74 +256,107 @@ export function ClusterView() {
|
||||
</span>
|
||||
}
|
||||
>
|
||||
<div className="flex flex-wrap gap-3 [&>*]:min-w-0 [&>*]:flex-[1_1_220px]">
|
||||
{members.map((node) => {
|
||||
const age = ageOf(node, now);
|
||||
const up = node.status === 'active';
|
||||
const share = age === null ? 1 : Math.min(1, age / STALE_AFTER_MS);
|
||||
return (
|
||||
<div
|
||||
key={node.hostname}
|
||||
className={cn(
|
||||
'relative overflow-hidden rounded-lg border bg-background/40 p-3.5',
|
||||
!up && 'border-destructive/50',
|
||||
)}
|
||||
>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="relative flex h-2.5 w-2.5">
|
||||
{up && (
|
||||
<span className="absolute inline-flex h-full w-full animate-ping rounded-full bg-primary opacity-50" />
|
||||
)}
|
||||
<span
|
||||
<div className="space-y-4">
|
||||
{grouped.map(([group, list]) => (
|
||||
<div key={group || '-'}>
|
||||
{group && (
|
||||
<div className="mb-2 flex items-baseline gap-2 text-xs">
|
||||
<span className="font-mono font-semibold">{group}</span>
|
||||
<span className="font-mono text-[10px] uppercase tracking-wider text-muted-foreground">
|
||||
{t('cc.groupCount', {
|
||||
count: list.length,
|
||||
up: list.filter((x) => x.status === 'active').length,
|
||||
defaultValue_one: '{{up}} of {{count}} pod up',
|
||||
defaultValue_other: '{{up}} of {{count}} pods up',
|
||||
})}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
<div className="flex flex-wrap gap-3 [&>*]:min-w-0 [&>*]:flex-[1_1_220px]">
|
||||
{list.map((node) => {
|
||||
const age = ageOf(node, now);
|
||||
const up = node.status === 'active';
|
||||
const share = age === null ? 1 : Math.min(1, age / STALE_AFTER_MS);
|
||||
return (
|
||||
<div
|
||||
key={node.hostname}
|
||||
className={cn(
|
||||
'relative inline-flex h-2.5 w-2.5 rounded-full',
|
||||
up ? 'bg-primary' : 'bg-destructive',
|
||||
'relative overflow-hidden rounded-lg border bg-background/40 p-3.5',
|
||||
!up && 'border-destructive/50',
|
||||
)}
|
||||
/>
|
||||
</span>
|
||||
<span className="truncate font-mono text-sm font-semibold">{node.hostname}</span>
|
||||
{node.nodeId !== undefined && (
|
||||
<span className="ml-auto font-mono text-[10px] uppercase tracking-wider text-muted-foreground">
|
||||
{t('cc.nodeId', 'id {{id}}', { id: node.nodeId })}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<div className="mt-2.5 flex items-baseline justify-between gap-2 whitespace-nowrap text-xs">
|
||||
<span className={cn('uppercase tracking-wider', up ? 'text-muted-foreground' : 'text-destructive')}>
|
||||
{up ? t('cc.nodeActive', 'Active') : t('cc.nodeSilent', 'Silent')}
|
||||
</span>
|
||||
<span className="font-mono tabular-nums">
|
||||
{age === null ? '—' : t('cc.heartbeatAgo', 'heartbeat {{ago}} ago', { ago: ago(age) })}
|
||||
</span>
|
||||
</div>
|
||||
<div className="mt-2 h-1 rounded-full bg-muted" aria-hidden>
|
||||
<div
|
||||
className={cn(
|
||||
'h-full rounded-full transition-[width] duration-1000 ease-linear',
|
||||
!up ? 'bg-destructive' : share > 0.5 ? 'bg-highlight' : 'bg-primary',
|
||||
)}
|
||||
style={{ width: `${Math.max(2, share * 100)}%` }}
|
||||
/>
|
||||
</div>
|
||||
{perNode && node.nodeId !== undefined && (
|
||||
<div className="mt-3 space-y-1.5 border-t pt-2.5">
|
||||
{nodeStrips(Number(node.nodeId)).map((row) => (
|
||||
<div key={row.label} className="grid grid-cols-[6.5rem_1fr] items-end gap-2">
|
||||
<div className="flex items-baseline justify-between gap-1 text-[10px] uppercase tracking-wider text-muted-foreground">
|
||||
<span className="truncate">{row.label}</span>
|
||||
<span className="font-mono text-xs font-semibold tabular-nums text-foreground">
|
||||
{n(row.total)}
|
||||
>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="relative flex h-2.5 w-2.5">
|
||||
{up && (
|
||||
<span className="absolute inline-flex h-full w-full animate-ping rounded-full bg-primary opacity-50" />
|
||||
)}
|
||||
<span
|
||||
className={cn(
|
||||
'relative inline-flex h-2.5 w-2.5 rounded-full',
|
||||
up ? 'bg-primary' : 'bg-destructive',
|
||||
)}
|
||||
/>
|
||||
</span>
|
||||
<span className="truncate font-mono text-sm font-semibold" title={node.hostname}>
|
||||
{group ? (podName(node.hostname)?.pod ?? node.hostname) : node.hostname}
|
||||
</span>
|
||||
{node.nodeId !== undefined && (
|
||||
<span className="ml-auto font-mono text-[10px] uppercase tracking-wider text-muted-foreground">
|
||||
{t('cc.nodeId', 'id {{id}}', { id: node.nodeId })}
|
||||
</span>
|
||||
</div>
|
||||
<BarStrip bars={row.bars} tone={up ? 'primary' : 'crit'} className="h-4" />
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
<div className="mt-2.5 flex items-baseline justify-between gap-2 whitespace-nowrap text-xs">
|
||||
<span
|
||||
className={cn('uppercase tracking-wider', up ? 'text-muted-foreground' : 'text-destructive')}
|
||||
>
|
||||
{up ? t('cc.nodeActive', 'Active') : t('cc.nodeSilent', 'Silent')}
|
||||
</span>
|
||||
<span className="font-mono tabular-nums">
|
||||
{age === null ? '—' : t('cc.heartbeatAgo', 'heartbeat {{ago}} ago', { ago: ago(age) })}
|
||||
</span>
|
||||
</div>
|
||||
<div className="mt-2 h-1 rounded-full bg-muted" aria-hidden>
|
||||
<div
|
||||
className={cn(
|
||||
'h-full rounded-full transition-[width] duration-1000 ease-linear',
|
||||
!up ? 'bg-destructive' : share > 0.5 ? 'bg-highlight' : 'bg-primary',
|
||||
)}
|
||||
style={{ width: `${Math.max(2, share * 100)}%` }}
|
||||
/>
|
||||
</div>
|
||||
{perNode && node.nodeId !== undefined && (
|
||||
<div className="mt-3 space-y-1.5 border-t pt-2.5">
|
||||
{nodeStrips(Number(node.nodeId)).map((row) => (
|
||||
<div key={row.label} className="grid grid-cols-[6.5rem_1fr] items-end gap-2">
|
||||
<div className="flex items-baseline justify-between gap-1 text-[10px] uppercase tracking-wider text-muted-foreground">
|
||||
<span className="truncate">{row.label}</span>
|
||||
<span className="font-mono text-xs font-semibold tabular-nums text-foreground">
|
||||
{n(row.total)}
|
||||
</span>
|
||||
</div>
|
||||
<BarStrip bars={row.bars} tone={up ? 'primary' : 'crit'} className="h-4" />
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
{health.gone.length > 0 && (
|
||||
<p className="mt-3 text-xs text-muted-foreground" title={health.gone.map((g) => g.hostname).join(', ')}>
|
||||
{t('cc.gone', {
|
||||
count: health.gone.length,
|
||||
defaultValue_one: '{{count}} pod replaced or finished, not counted: {{names}}',
|
||||
defaultValue_other: '{{count}} pods replaced or finished, not counted: {{names}}',
|
||||
names: [...goneByGroup.entries()].map(([g, c]) => (c > 1 ? `${g} ×${c}` : g)).join(', '),
|
||||
})}
|
||||
</p>
|
||||
)}
|
||||
{retired.length > 0 && (
|
||||
<p className="mt-3 text-xs text-muted-foreground">
|
||||
{t('cc.retired', {
|
||||
|
||||
+5
-1
@@ -1990,7 +1990,11 @@
|
||||
"zoomed": "Zoom",
|
||||
"dragToZoom": "Drag to zoom",
|
||||
"nodeMail": "Mail",
|
||||
"nodeConnections": "Connections"
|
||||
"nodeConnections": "Connections",
|
||||
"groupCount_one": "{{up}} of {{count}} pod up",
|
||||
"groupCount_other": "{{up}} of {{count}} pods up",
|
||||
"gone_one": "{{count}} pod replaced or finished, not counted: {{names}}",
|
||||
"gone_other": "{{count}} pods replaced or finished, not counted: {{names}}"
|
||||
},
|
||||
"commands": {
|
||||
"actionHint": "Server action",
|
||||
|
||||
Reference in new issue
Block a user