Cluster Health: readable on Kubernetes, no pile-up after rollouts (K-A3) #258

Merged
jcoffey-dev merged 1 commits from feat/cluster-health-pods into main 2026-11-20 07:33:25 +00:00
4 changed files with 241 additions and 72 deletions

No files matched your search

+62 -3
View File
@@ -5,7 +5,7 @@
*/
import { describe, expect, it } from 'vitest';
import { minutesSilent, setOf, summarizeNodes, unboundOf, type ClusterNodeRow } from './clusterHealth';
import { minutesSilent, podName, setOf, summarizeNodes, unboundOf, type ClusterNodeRow } from './clusterHealth';
const node = (hostname: string, status: ClusterNodeRow['status'], lastRenewal = '2026-09-25T10:00:00Z') => ({
hostname,
@@ -27,7 +27,7 @@ describe('summarizeNodes', () => {
node('mx2.example.org', 'active'),
node('mx3.example.org', 'active'),
]),
).toEqual({ healthy: 3, unhealthy: 0, silent: [] });
).toEqual({ healthy: 3, unhealthy: 0, silent: [], gone: [] });
});
it('names the silent nodes, longest quiet first', () => {
@@ -53,6 +53,63 @@ describe('summarizeNodes', () => {
});
});
describe('pods (K-A3)', () => {
it('reads Kubernetes pod names, and nothing else', () => {
expect(podName('inbuxa-mta-674c7dffc5-c9bp7')).toEqual({
group: 'inbuxa-mta',
pod: '674c7dffc5-c9bp7',
replicated: true,
});
expect(podName('inbuxa-upgrade-fcxxt')).toEqual({ group: 'inbuxa-upgrade', pod: 'fcxxt', replicated: false });
expect(podName('mx2.example.org')).toBeNull();
expect(podName('mail')).toBeNull();
expect(podName('inbuxa-0')).toBeNull(); // a StatefulSet's pod keeps its name
expect(podName('mail-relay')).toBeNull(); // vowels: a word, not a generated suffix
});
// The lab after a few rollouts: five pods renewing, four replaced, and the
// upgrade hook's pod that ran once.
const lab = [
node('inbuxa-core-786fbfd875-wftz8', 'active', '2026-11-20T07:24:47Z'),
node('inbuxa-front-75d4794556-9nv8k', 'active', '2026-11-20T07:24:45Z'),
node('inbuxa-front-75d4794556-x7tjq', 'active', '2026-11-20T07:24:49Z'),
node('inbuxa-mta-674c7dffc5-c9bp7', 'active', '2026-11-20T07:24:45Z'),
node('inbuxa-mta-674c7dffc5-jj6br', 'active', '2026-11-20T07:24:47Z'),
node('inbuxa-front-75d4794556-4m47f', 'stale', '2026-11-20T06:49:40Z'),
node('inbuxa-front-75d4794556-v2zxq', 'stale', '2026-11-20T06:50:03Z'),
node('inbuxa-mta-674c7dffc5-5f6v9', 'stale', '2026-11-20T06:49:49Z'),
node('inbuxa-mta-674c7dffc5-9g5nx', 'stale', '2026-11-20T06:50:03Z'),
node('inbuxa-upgrade-fcxxt', 'stale', '2026-11-20T07:19:22Z'),
];
it('does not count pods a rollout replaced, or a finished Job', () => {
const health = summarizeNodes(lab);
expect(health).toMatchObject({ healthy: 5, unhealthy: 0, silent: [] });
expect(health?.gone.map((n) => n.hostname)).toEqual([
'inbuxa-upgrade-fcxxt',
'inbuxa-front-75d4794556-v2zxq',
'inbuxa-mta-674c7dffc5-9g5nx',
'inbuxa-mta-674c7dffc5-5f6v9',
'inbuxa-front-75d4794556-4m47f',
]);
});
it('counts a role with no pod renewing as down, once', () => {
const health = summarizeNodes(
lab.filter((n) => !n.hostname.startsWith('inbuxa-mta-674c7dffc5-') || n.status !== 'active'),
);
expect(health).toMatchObject({ healthy: 3, unhealthy: 1 });
expect(health?.silent.map((n) => n.hostname)).toEqual(['inbuxa-mta-674c7dffc5-9g5nx']);
expect(health?.gone.map((n) => n.hostname)).toContain('inbuxa-mta-674c7dffc5-5f6v9');
});
it('still counts a silent server that is not a pod', () => {
const health = summarizeNodes([...lab, node('mx2.example.org', 'stale', '2026-11-20T07:00:00Z')]);
expect(health).toMatchObject({ healthy: 5, unhealthy: 1 });
expect(health?.silent.map((n) => n.hostname)).toEqual(['mx2.example.org']);
});
});
describe('minutesSilent', () => {
it('counts whole minutes since the last renewal', () => {
const now = Date.parse('2026-09-25T10:07:30Z');
@@ -77,7 +134,9 @@ describe('unboundOf (inbuxa-server#485)', () => {
describe('setOf', () => {
it('reads a registry set as the server sends it', () => {
// As a live server answered for a listener held by another process
const wire = { 'imaps [::]:12993: Failed to bind to [::]:12993: Address already in use (os error 98); retrying': true };
const wire = {
'imaps [::]:12993: Failed to bind to [::]:12993: Address already in use (os error 98); retrying': true,
};
expect(setOf(wire)).toEqual([
'imaps [::]:12993: Failed to bind to [::]:12993: Address already in use (os error 98); retrying',
]);
+63 -4
View File
@@ -46,21 +46,80 @@ export function unboundOf(nodes: ClusterNodeRow[]): UnboundNode[] {
.map(({ hostname, unboundListeners }) => ({ hostname, listeners: unboundListeners }));
}
/*
* inbuxa: K-A3. On Kubernetes a node's hostname is its pod's name, and the
* name says what made the pod: a Deployment's pods are named
* `<deployment>-<template hash>-<5>`, a Job's (and a DaemonSet's)
* `<job>-<5>`, the generated parts drawn from an alphabet with no vowels and
* no 0, 1 or 3. A pod's name is never used again: one that stopped renewing
* isn't coming back, its replacement leases under a new name. So a silent
* lease with a pod's name is a pod that's gone (a rollout replaced it, or a
* Job finished), not a node that's down, and it's left out of the count,
* which would otherwise be half red for a day after every rollout. Only
* when no pod of a Deployment is renewing at all is the role down, and then
* it counts once.
*/
const POD_CHARS = '[bcdfghjklmnpqrstvwxz2456789]';
const DEPLOYMENT_POD = new RegExp(`^([a-z0-9][a-z0-9-]*?)-(${POD_CHARS}{6,10})-(${POD_CHARS}{5})$`);
const OTHER_POD = new RegExp(`^([a-z0-9][a-z0-9-]*?)-(${POD_CHARS}{5})$`);
export interface PodName {
/** What made the pod: the Deployment, Job or DaemonSet's name. */
group: string;
/** The generated part that tells this pod from its siblings. */
pod: string;
/** A Deployment's pod, which is replaced when it goes. */
replicated: boolean;
}
/** The parts of a Kubernetes pod name, or null for any other hostname. */
export function podName(hostname: string): PodName | null {
const d = DEPLOYMENT_POD.exec(hostname);
if (d) return { group: d[1], pod: `${d[2]}-${d[3]}`, replicated: true };
const o = OTHER_POD.exec(hostname);
return o ? { group: o[1], pod: o[2], replicated: false } : null;
}
export interface ClusterHealth {
healthy: number;
unhealthy: number;
/** Nodes that stopped renewing, longest silent first. */
silent: { hostname: string; lastRenewal: string }[];
/** Pods that stopped renewing and won't be back (see podName), latest first. */
gone: { hostname: string; lastRenewal: string }[];
}
export function summarizeNodes(nodes: ClusterNodeRow[]): ClusterHealth | null {
const members = nodes.filter((n) => n.status !== 'inactive');
if (members.length < 2) return null;
const silent = members
.filter((n) => n.status !== 'active')
const quiet = members.filter((n) => n.status !== 'active');
const renewing = new Set(
members
.filter((n) => n.status === 'active')
.map((n) => podName(n.hostname)?.group)
.filter(Boolean),
);
// A Deployment with no pod renewing: its latest pod stands for it
const down = new Map<string, ClusterNodeRow>();
for (const n of quiet) {
const pod = podName(n.hostname);
if (!pod?.replicated || renewing.has(pod.group)) continue;
const seen = down.get(pod.group);
if (!seen || seen.lastRenewal < n.lastRenewal) down.set(pod.group, n);
}
const standsFor = new Set(down.values());
const isGone = (n: ClusterNodeRow) => podName(n.hostname) !== null && !standsFor.has(n);
const pick = ({ hostname, lastRenewal }: ClusterNodeRow) => ({ hostname, lastRenewal });
const silent = quiet
.filter((n) => !isGone(n))
.sort((a, b) => a.lastRenewal.localeCompare(b.lastRenewal))
.map(({ hostname, lastRenewal }) => ({ hostname, lastRenewal }));
return { healthy: members.length - silent.length, unhealthy: silent.length, silent };
.map(pick);
const gone = quiet
.filter(isGone)
.sort((a, b) => b.lastRenewal.localeCompare(a.lastRenewal))
.map(pick);
const healthy = members.length - quiet.length;
return { healthy, unhealthy: silent.length, silent, gone };
}
/** Whole minutes since a node last renewed its lease, or null if the time can't be read. */
+111 -64
View File
@@ -27,7 +27,7 @@ import {
series,
total,
} from '../commandCenter';
import type { ClusterNodeRow } from '../clusterHealth';
import { podName, type ClusterNodeRow } from '../clusterHealth';
import { useDashboardShared } from '../dashboardContext';
import { usePageHistory } from '../usePageHistory';
import type { Tone } from '../tones';
@@ -130,8 +130,22 @@ export function ClusterView() {
);
const { nodes, health } = cluster;
const members = nodes.filter((n) => n.status !== 'inactive');
// inbuxa: K-A3, pods that are gone (see podName) get a line, not a card
const gone = new Set(health.gone.map((g) => g.hostname));
const members = nodes.filter((n) => n.status !== 'inactive' && !gone.has(n.hostname));
const retired = nodes.filter((n) => n.status === 'inactive');
// Cards grouped by what made the pod; servers that aren't pods come first
const groups = new Map<string, ClusterNodeRow[]>();
for (const node of [...members].sort((a, b) => a.hostname.localeCompare(b.hostname))) {
const key = podName(node.hostname)?.group ?? '';
groups.set(key, [...(groups.get(key) ?? []), node]);
}
const grouped = [...groups.entries()].sort(([a], [b]) => (a === '' ? -1 : b === '' ? 1 : a.localeCompare(b)));
const goneByGroup = new Map<string, number>();
for (const g of health.gone) {
const key = podName(g.hostname)?.group ?? g.hostname;
goneByGroup.set(key, (goneByGroup.get(key) ?? 0) + 1);
}
const all = health.healthy + health.unhealthy;
const activeAges = members
.filter((n) => n.status === 'active')
@@ -242,74 +256,107 @@ export function ClusterView() {
</span>
}
>
<div className="flex flex-wrap gap-3 [&>*]:min-w-0 [&>*]:flex-[1_1_220px]">
{members.map((node) => {
const age = ageOf(node, now);
const up = node.status === 'active';
const share = age === null ? 1 : Math.min(1, age / STALE_AFTER_MS);
return (
<div
key={node.hostname}
className={cn(
'relative overflow-hidden rounded-lg border bg-background/40 p-3.5',
!up && 'border-destructive/50',
)}
>
<div className="flex items-center gap-2">
<span className="relative flex h-2.5 w-2.5">
{up && (
<span className="absolute inline-flex h-full w-full animate-ping rounded-full bg-primary opacity-50" />
)}
<span
<div className="space-y-4">
{grouped.map(([group, list]) => (
<div key={group || '-'}>
{group && (
<div className="mb-2 flex items-baseline gap-2 text-xs">
<span className="font-mono font-semibold">{group}</span>
<span className="font-mono text-[10px] uppercase tracking-wider text-muted-foreground">
{t('cc.groupCount', {
count: list.length,
up: list.filter((x) => x.status === 'active').length,
defaultValue_one: '{{up}} of {{count}} pod up',
defaultValue_other: '{{up}} of {{count}} pods up',
})}
</span>
</div>
)}
<div className="flex flex-wrap gap-3 [&>*]:min-w-0 [&>*]:flex-[1_1_220px]">
{list.map((node) => {
const age = ageOf(node, now);
const up = node.status === 'active';
const share = age === null ? 1 : Math.min(1, age / STALE_AFTER_MS);
return (
<div
key={node.hostname}
className={cn(
'relative inline-flex h-2.5 w-2.5 rounded-full',
up ? 'bg-primary' : 'bg-destructive',
'relative overflow-hidden rounded-lg border bg-background/40 p-3.5',
!up && 'border-destructive/50',
)}
/>
</span>
<span className="truncate font-mono text-sm font-semibold">{node.hostname}</span>
{node.nodeId !== undefined && (
<span className="ml-auto font-mono text-[10px] uppercase tracking-wider text-muted-foreground">
{t('cc.nodeId', 'id {{id}}', { id: node.nodeId })}
</span>
)}
</div>
<div className="mt-2.5 flex items-baseline justify-between gap-2 whitespace-nowrap text-xs">
<span className={cn('uppercase tracking-wider', up ? 'text-muted-foreground' : 'text-destructive')}>
{up ? t('cc.nodeActive', 'Active') : t('cc.nodeSilent', 'Silent')}
</span>
<span className="font-mono tabular-nums">
{age === null ? '—' : t('cc.heartbeatAgo', 'heartbeat {{ago}} ago', { ago: ago(age) })}
</span>
</div>
<div className="mt-2 h-1 rounded-full bg-muted" aria-hidden>
<div
className={cn(
'h-full rounded-full transition-[width] duration-1000 ease-linear',
!up ? 'bg-destructive' : share > 0.5 ? 'bg-highlight' : 'bg-primary',
)}
style={{ width: `${Math.max(2, share * 100)}%` }}
/>
</div>
{perNode && node.nodeId !== undefined && (
<div className="mt-3 space-y-1.5 border-t pt-2.5">
{nodeStrips(Number(node.nodeId)).map((row) => (
<div key={row.label} className="grid grid-cols-[6.5rem_1fr] items-end gap-2">
<div className="flex items-baseline justify-between gap-1 text-[10px] uppercase tracking-wider text-muted-foreground">
<span className="truncate">{row.label}</span>
<span className="font-mono text-xs font-semibold tabular-nums text-foreground">
{n(row.total)}
>
<div className="flex items-center gap-2">
<span className="relative flex h-2.5 w-2.5">
{up && (
<span className="absolute inline-flex h-full w-full animate-ping rounded-full bg-primary opacity-50" />
)}
<span
className={cn(
'relative inline-flex h-2.5 w-2.5 rounded-full',
up ? 'bg-primary' : 'bg-destructive',
)}
/>
</span>
<span className="truncate font-mono text-sm font-semibold" title={node.hostname}>
{group ? (podName(node.hostname)?.pod ?? node.hostname) : node.hostname}
</span>
{node.nodeId !== undefined && (
<span className="ml-auto font-mono text-[10px] uppercase tracking-wider text-muted-foreground">
{t('cc.nodeId', 'id {{id}}', { id: node.nodeId })}
</span>
</div>
<BarStrip bars={row.bars} tone={up ? 'primary' : 'crit'} className="h-4" />
)}
</div>
))}
</div>
)}
<div className="mt-2.5 flex items-baseline justify-between gap-2 whitespace-nowrap text-xs">
<span
className={cn('uppercase tracking-wider', up ? 'text-muted-foreground' : 'text-destructive')}
>
{up ? t('cc.nodeActive', 'Active') : t('cc.nodeSilent', 'Silent')}
</span>
<span className="font-mono tabular-nums">
{age === null ? '—' : t('cc.heartbeatAgo', 'heartbeat {{ago}} ago', { ago: ago(age) })}
</span>
</div>
<div className="mt-2 h-1 rounded-full bg-muted" aria-hidden>
<div
className={cn(
'h-full rounded-full transition-[width] duration-1000 ease-linear',
!up ? 'bg-destructive' : share > 0.5 ? 'bg-highlight' : 'bg-primary',
)}
style={{ width: `${Math.max(2, share * 100)}%` }}
/>
</div>
{perNode && node.nodeId !== undefined && (
<div className="mt-3 space-y-1.5 border-t pt-2.5">
{nodeStrips(Number(node.nodeId)).map((row) => (
<div key={row.label} className="grid grid-cols-[6.5rem_1fr] items-end gap-2">
<div className="flex items-baseline justify-between gap-1 text-[10px] uppercase tracking-wider text-muted-foreground">
<span className="truncate">{row.label}</span>
<span className="font-mono text-xs font-semibold tabular-nums text-foreground">
{n(row.total)}
</span>
</div>
<BarStrip bars={row.bars} tone={up ? 'primary' : 'crit'} className="h-4" />
</div>
))}
</div>
)}
</div>
);
})}
</div>
);
})}
</div>
))}
</div>
{health.gone.length > 0 && (
<p className="mt-3 text-xs text-muted-foreground" title={health.gone.map((g) => g.hostname).join(', ')}>
{t('cc.gone', {
count: health.gone.length,
defaultValue_one: '{{count}} pod replaced or finished, not counted: {{names}}',
defaultValue_other: '{{count}} pods replaced or finished, not counted: {{names}}',
names: [...goneByGroup.entries()].map(([g, c]) => (c > 1 ? `${g} ×${c}` : g)).join(', '),
})}
</p>
)}
{retired.length > 0 && (
<p className="mt-3 text-xs text-muted-foreground">
{t('cc.retired', {
+5 -1
View File
@@ -1990,7 +1990,11 @@
"zoomed": "Zoom",
"dragToZoom": "Drag to zoom",
"nodeMail": "Mail",
"nodeConnections": "Connections"
"nodeConnections": "Connections",
"groupCount_one": "{{up}} of {{count}} pod up",
"groupCount_other": "{{up}} of {{count}} pods up",
"gone_one": "{{count}} pod replaced or finished, not counted: {{names}}",
"gone_other": "{{count}} pods replaced or finished, not counted: {{names}}"
},
"commands": {
"actionHint": "Server action",