Phase 3: dashboards and alerting
Saved, shareable multi-panel dashboards (table/line/bar/single-stat panels via gridstack + uPlot, global + per-panel time range, JSON export/import) and threshold/absence alert rules with an ok/pending/firing evaluator and webhook/Slack/PagerDuty delivery. - New /metadata component: Postgres control-plane store for dashboards, panels, notification targets, alert rules/state, and delivery log -- see docs/phase-3-dashboard-design.md for why ClickHouse's MergeTree family isn't a fit for this access pattern (needs real row-level locking and read-your-writes consistency). - api/internal/dashboards: dashboard/panel CRUD, pure -- panel query execution stays client-side, reusing the existing /query endpoint. - New /alerting service: rule/target CRUD, a ticker-driven evaluator (claim-then-evaluate concurrency control, transactional-outbox delivery, query errors and threshold zero-rows never coerced into a false transition) and webhook/Slack/PagerDuty delivery with retry/backoff. See docs/phase-3-alerting-design.md for the full state-machine design and the four correctness properties it implements. - web: /dashboards and /alerts UIs; cli: sentryctl dashboards/alerts list/get/apply, seeding a future Terraform provider's JSON contract. - hack/alert-load-test: 500 rules against real ClickHouse data, real measured results in docs/phase-3-runbook.md. Five real bugs found by actually running this against a live stack (documented in the runbook, not just fixed silently): a latent Phase 2 bug where ClickHouse rejected the timestamp format used for earliest=/latest= queries; a "now" literal token injected into query text; a GridStack/uPlot layout-timing race; JS's Date.parse being too lenient to use as a timestamp-detection heuristic; a rule's "enabled" field silently defaulting to false when omitted; and the evaluator's claim-batch-size and worker-pool-concurrency defaulting to the same value, causing 500 concurrently-due rules to take 125s to cycle through instead of the configured 60s.
This commit is contained in:
@@ -0,0 +1,147 @@
|
||||
<script lang="ts">
|
||||
// Dispatches a query result to the right rendering for a panel's
|
||||
// viz_type. table/top_n reuse ResultsTable.svelte (top_n is "table,
|
||||
// but the query already did sort/head" -- same execution path per
|
||||
// /docs/phase-3-dashboard-design.md). line/bar use uPlot.
|
||||
import uPlot from 'uplot';
|
||||
import 'uplot/dist/uPlot.min.css';
|
||||
import ResultsTable from '$lib/ResultsTable.svelte';
|
||||
import type { QueryResult, VizType } from '$lib/api';
|
||||
|
||||
let {
|
||||
result,
|
||||
vizType,
|
||||
vizConfig = {}
|
||||
}: { result: QueryResult; vizType: VizType; vizConfig?: Record<string, string> } = $props();
|
||||
|
||||
let chartEl: HTMLDivElement | undefined = $state();
|
||||
let chart: uPlot | undefined;
|
||||
|
||||
// ISO-8601-ish prefix ("2026-08-13T20:20:24...") -- the shape Sentry's
|
||||
// own timestamp column actually comes back as. Deliberately narrow:
|
||||
// found by actually rendering a `stats count by host` bar chart that
|
||||
// JS's built-in Date.parse() is far too lenient to use as a "does
|
||||
// this look like a timestamp" check -- Date.parse("host-06") returns
|
||||
// a real (bogus) timestamp rather than NaN, which silently misrouted
|
||||
// a categorical `host` column onto a numeric time axis and rendered
|
||||
// unreadable giant tick labels instead of host names.
|
||||
const isoTimestampPrefix = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/;
|
||||
|
||||
// Prefers a real numeric/time x-axis when the column parses cleanly
|
||||
// (e.g. a timestamp column); falls back to row index with the raw
|
||||
// value as a tick label otherwise (e.g. a `stats ... by host` grouping
|
||||
// column, which is categorical text).
|
||||
function resolveXAxis(columns: string[], rows: unknown[][], columnName: string | undefined) {
|
||||
const idx = columnName ? Math.max(columns.indexOf(columnName), 0) : 0;
|
||||
const labels = rows.map((r) => String(r[idx] ?? ''));
|
||||
const asSeconds = rows.map((r) => {
|
||||
const v = r[idx];
|
||||
if (typeof v === 'number') return v;
|
||||
if (typeof v !== 'string' || !isoTimestampPrefix.test(v)) return NaN;
|
||||
const parsed = Date.parse(v);
|
||||
return Number.isNaN(parsed) ? NaN : parsed / 1000;
|
||||
});
|
||||
const allNumeric = asSeconds.every((n) => !Number.isNaN(n));
|
||||
return allNumeric
|
||||
? { values: asSeconds, labels, categorical: false }
|
||||
: { values: rows.map((_, i) => i), labels, categorical: true };
|
||||
}
|
||||
|
||||
function resolveValueColumn(columns: string[], columnName: string | undefined): number {
|
||||
if (columnName) {
|
||||
const i = columns.indexOf(columnName);
|
||||
if (i >= 0) return i;
|
||||
}
|
||||
// default: second column if there is one (first is usually the
|
||||
// grouping/x column), otherwise the only column there is.
|
||||
return columns.length > 1 ? 1 : 0;
|
||||
}
|
||||
|
||||
function renderChart() {
|
||||
if (!chartEl) return;
|
||||
chart?.destroy();
|
||||
chart = undefined;
|
||||
if (vizType !== 'line' && vizType !== 'bar') return;
|
||||
|
||||
const { columns, rows } = result;
|
||||
if (columns.length === 0 || rows.length === 0) return;
|
||||
|
||||
const x = resolveXAxis(columns, rows, vizConfig.x_column);
|
||||
const valueIdx = resolveValueColumn(columns, vizConfig.value_column);
|
||||
const values = rows.map((r) => {
|
||||
const v = r[valueIdx];
|
||||
return typeof v === 'number' ? v : Number(v) || 0;
|
||||
});
|
||||
|
||||
chart = new uPlot(
|
||||
{
|
||||
width: chartEl.clientWidth || 400,
|
||||
height: 220,
|
||||
legend: { show: false },
|
||||
series: [
|
||||
{},
|
||||
{
|
||||
label: columns[valueIdx],
|
||||
stroke: '#06c',
|
||||
fill: vizType === 'bar' ? '#06c33' : undefined,
|
||||
width: vizType === 'bar' ? 0 : 2,
|
||||
paths: vizType === 'bar' ? uPlot.paths.bars!({ size: [0.6] }) : undefined
|
||||
}
|
||||
],
|
||||
axes: [
|
||||
{
|
||||
values: (_u, ticks) =>
|
||||
ticks.map((t) => (x.categorical ? (x.labels[t] ?? '') : String(t)))
|
||||
},
|
||||
{}
|
||||
]
|
||||
},
|
||||
[x.values, values],
|
||||
chartEl
|
||||
);
|
||||
}
|
||||
|
||||
$effect(() => {
|
||||
// Re-render whenever the result or viz settings change.
|
||||
result;
|
||||
vizType;
|
||||
vizConfig;
|
||||
renderChart();
|
||||
return () => chart?.destroy();
|
||||
});
|
||||
|
||||
// A dashboard panel's real width isn't known at first render --
|
||||
// gridstack.js sizes the parent .grid-stack-item via its own layout
|
||||
// pass, which can land after this component's own effect runs. Found
|
||||
// by actually adding a bar-chart panel and inspecting the rendered
|
||||
// canvas: it came out 74px wide (chartEl.clientWidth measured before
|
||||
// gridstack finished sizing the container), not the container's real
|
||||
// ~540px. A ResizeObserver re-renders whenever chartEl's actual size
|
||||
// changes, which fixes both that initial race and, as a side benefit,
|
||||
// keeps the chart correctly sized when a panel is drag-resized later.
|
||||
$effect(() => {
|
||||
if (!chartEl) return;
|
||||
const observer = new ResizeObserver(() => renderChart());
|
||||
observer.observe(chartEl);
|
||||
return () => observer.disconnect();
|
||||
});
|
||||
</script>
|
||||
|
||||
{#if vizType === 'table' || vizType === 'top_n'}
|
||||
<ResultsTable columns={result.columns} rows={result.rows} hasRun={true} />
|
||||
{:else if vizType === 'single_stat'}
|
||||
<div class="single-stat">{result.rows[0]?.[0] ?? '—'}</div>
|
||||
{:else}
|
||||
<div bind:this={chartEl} class="chart"></div>
|
||||
{/if}
|
||||
|
||||
<style>
|
||||
.single-stat {
|
||||
font-size: 2.5rem;
|
||||
font-weight: 600;
|
||||
padding: 1rem 0;
|
||||
}
|
||||
.chart {
|
||||
width: 100%;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,93 @@
|
||||
<script lang="ts">
|
||||
// Extracted from the root query page in Phase 3 so the dashboard panel
|
||||
// editor and the alert rule editor can reuse the same input --
|
||||
// deliberately just the input+run affordance, not results/history,
|
||||
// which differ per consumer.
|
||||
import type { Language } from '$lib/api';
|
||||
|
||||
let {
|
||||
query = $bindable(''),
|
||||
language = $bindable<Language>(''),
|
||||
onRun,
|
||||
loading = false,
|
||||
placeholder = 'service=api | where status>=500 | stats count by host | sort -count'
|
||||
}: {
|
||||
query: string;
|
||||
language: Language;
|
||||
onRun: () => void;
|
||||
loading?: boolean;
|
||||
placeholder?: string;
|
||||
} = $props();
|
||||
|
||||
// Client-side mirror of the backend's auto-detect heuristic
|
||||
// (api/internal/querylang/planner.looksLikeSQL) -- purely a UI hint,
|
||||
// the server does its own detection independently and is the
|
||||
// authority on what actually runs.
|
||||
function detectedLanguage(q: string): 'sql' | 'spl' {
|
||||
return /^\s*select\b/i.test(q) ? 'sql' : 'spl';
|
||||
}
|
||||
let detected = $derived(detectedLanguage(query));
|
||||
let effectiveLanguage = $derived(language === '' ? detected : language);
|
||||
|
||||
function onKeydown(e: KeyboardEvent) {
|
||||
// Cmd/Ctrl+Enter runs the query -- textarea's own Enter key needs
|
||||
// to stay newline-for-pipe-stage-formatting, so this isn't a bare
|
||||
// Enter binding.
|
||||
if ((e.metaKey || e.ctrlKey) && e.key === 'Enter') {
|
||||
e.preventDefault();
|
||||
onRun();
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="query-bar">
|
||||
<textarea bind:value={query} onkeydown={onKeydown} rows="4" spellcheck="false" {placeholder}></textarea>
|
||||
<div class="controls">
|
||||
<label>
|
||||
Language:
|
||||
<select bind:value={language}>
|
||||
<option value="">Auto ({detected})</option>
|
||||
<option value="spl">Pipe syntax</option>
|
||||
<option value="sql">SQL</option>
|
||||
</select>
|
||||
</label>
|
||||
<span class="detected-badge" class:sql={effectiveLanguage === 'sql'}>
|
||||
{effectiveLanguage === 'sql' ? 'SQL' : 'pipe syntax'}
|
||||
</span>
|
||||
<button onclick={onRun} disabled={loading || query.trim() === ''}>
|
||||
{loading ? 'Running…' : 'Run query'}
|
||||
</button>
|
||||
<span class="hint">⌘/Ctrl+Enter to run</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<style>
|
||||
.query-bar textarea {
|
||||
width: 100%;
|
||||
font-family: monospace;
|
||||
font-size: 0.9rem;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
.controls {
|
||||
margin-top: 0.5rem;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.75rem;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
.detected-badge {
|
||||
font-size: 0.75rem;
|
||||
padding: 0.15rem 0.5rem;
|
||||
border-radius: 1rem;
|
||||
background: #eef;
|
||||
color: #224;
|
||||
}
|
||||
.detected-badge.sql {
|
||||
background: #fee;
|
||||
color: #422;
|
||||
}
|
||||
.hint {
|
||||
font-size: 0.8rem;
|
||||
color: #777;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,242 @@
|
||||
// First real API-client module -- previously each route did its own
|
||||
// inline fetch(). Introduced in Phase 3 because the surface triples
|
||||
// (query + dashboards + panels + export/import); still zero-dependency,
|
||||
// a thin fetch wrapper, not a generated client.
|
||||
|
||||
export const apiBase = import.meta.env.VITE_API_BASE_URL ?? 'http://localhost:8080';
|
||||
export const alertingBase = import.meta.env.VITE_ALERTING_API_BASE_URL ?? 'http://localhost:8081';
|
||||
|
||||
export type Language = '' | 'sql' | 'spl';
|
||||
|
||||
export type QueryResult = { columns: string[]; rows: unknown[][] };
|
||||
|
||||
export type VizType = 'table' | 'line' | 'bar' | 'single_stat' | 'top_n';
|
||||
|
||||
export type Panel = {
|
||||
id: string;
|
||||
dashboard_id: string;
|
||||
title: string;
|
||||
query: string;
|
||||
query_language: Language;
|
||||
viz_type: VizType;
|
||||
viz_config: Record<string, string>;
|
||||
position_x: number;
|
||||
position_y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
earliest_override: string | null;
|
||||
latest_override: string | null;
|
||||
sort_order: number;
|
||||
};
|
||||
|
||||
export type Dashboard = {
|
||||
id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
default_earliest: string;
|
||||
default_latest: string;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
panels: Panel[] | null;
|
||||
};
|
||||
|
||||
class ApiError extends Error {}
|
||||
|
||||
async function requestFrom<T>(base: string, path: string, init?: RequestInit): Promise<T> {
|
||||
const res = await fetch(`${base}${path}`, {
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
...init
|
||||
});
|
||||
if (!res.ok) {
|
||||
let message = `request failed with status ${res.status}`;
|
||||
try {
|
||||
const body = await res.json();
|
||||
if (body?.error) message = body.error;
|
||||
} catch {
|
||||
// non-JSON error body -- keep the generic message
|
||||
}
|
||||
throw new ApiError(message);
|
||||
}
|
||||
if (res.status === 204) return undefined as T;
|
||||
return res.json();
|
||||
}
|
||||
|
||||
function request<T>(path: string, init?: RequestInit): Promise<T> {
|
||||
return requestFrom(apiBase, path, init);
|
||||
}
|
||||
|
||||
// alerting is a separate service (its own base URL) -- see
|
||||
// /docs/phase-3-alerting-design.md's component boundary.
|
||||
function alertingRequest<T>(path: string, init?: RequestInit): Promise<T> {
|
||||
return requestFrom(alertingBase, path, init);
|
||||
}
|
||||
|
||||
export function runQuery(query: string, language: Language): Promise<QueryResult> {
|
||||
return request('/query', { method: 'POST', body: JSON.stringify({ query, language }) });
|
||||
}
|
||||
|
||||
export function listDashboards(): Promise<Dashboard[]> {
|
||||
return request('/dashboards').then((d) => (d as Dashboard[]) ?? []);
|
||||
}
|
||||
|
||||
export function getDashboard(id: string): Promise<Dashboard> {
|
||||
return request(`/dashboards/${id}`);
|
||||
}
|
||||
|
||||
export function createDashboard(input: {
|
||||
name: string;
|
||||
description?: string;
|
||||
default_earliest?: string;
|
||||
default_latest?: string;
|
||||
}): Promise<Dashboard> {
|
||||
return request('/dashboards', { method: 'POST', body: JSON.stringify(input) });
|
||||
}
|
||||
|
||||
export function updateDashboard(
|
||||
id: string,
|
||||
input: { name: string; description?: string; default_earliest?: string; default_latest?: string }
|
||||
): Promise<Dashboard> {
|
||||
return request(`/dashboards/${id}`, { method: 'PUT', body: JSON.stringify(input) });
|
||||
}
|
||||
|
||||
export function deleteDashboard(id: string): Promise<void> {
|
||||
return request(`/dashboards/${id}`, { method: 'DELETE' });
|
||||
}
|
||||
|
||||
export function addPanel(dashboardId: string, panel: Partial<Panel>): Promise<Panel> {
|
||||
return request(`/dashboards/${dashboardId}/panels`, { method: 'POST', body: JSON.stringify(panel) });
|
||||
}
|
||||
|
||||
export function updatePanel(dashboardId: string, panel: Partial<Panel>): Promise<Panel> {
|
||||
return request(`/dashboards/${dashboardId}/panels/${panel.id}`, {
|
||||
method: 'PUT',
|
||||
body: JSON.stringify(panel)
|
||||
});
|
||||
}
|
||||
|
||||
export function deletePanel(dashboardId: string, panelId: string): Promise<void> {
|
||||
return request(`/dashboards/${dashboardId}/panels/${panelId}`, { method: 'DELETE' });
|
||||
}
|
||||
|
||||
export function exportDashboard(id: string): Promise<Dashboard> {
|
||||
return request(`/dashboards/${id}/export`);
|
||||
}
|
||||
|
||||
export function importDashboard(dashboard: Dashboard): Promise<Dashboard> {
|
||||
return request('/dashboards/import', { method: 'POST', body: JSON.stringify(dashboard) });
|
||||
}
|
||||
|
||||
// resolveTimeRange applies the override-or-default rule from
|
||||
// /docs/phase-3-dashboard-design.md's "Time-range mechanics": a panel's
|
||||
// own earliest/latest override wins if set, otherwise the dashboard's
|
||||
// default applies.
|
||||
export function resolveTimeRange(
|
||||
dashboard: Pick<Dashboard, 'default_earliest' | 'default_latest'>,
|
||||
panel: Pick<Panel, 'earliest_override' | 'latest_override'>
|
||||
): { earliest: string; latest: string } {
|
||||
return {
|
||||
earliest: panel.earliest_override ?? dashboard.default_earliest,
|
||||
latest: panel.latest_override ?? dashboard.default_latest
|
||||
};
|
||||
}
|
||||
|
||||
// injectTimeRange prepends earliest=/latest= as leading base_search
|
||||
// terms -- works because they're ordinary implicit-AND terms in Phase
|
||||
// 2's grammar, order-independent. Never used for raw-SQL panels (the
|
||||
// dashboards API rejects query_language: "sql" on panels entirely, so
|
||||
// this never has to handle that case).
|
||||
//
|
||||
// "now" is a UI-only sentinel (the default_latest value shown in the
|
||||
// time-range picker), not a token the query language understands --
|
||||
// time_expr only accepts a quoted absolute timestamp or a "-N unit"
|
||||
// relative offset (see /docs/query-language-design.md). Emitting a
|
||||
// literal `latest=now` produces a real compile error ("expected a
|
||||
// quoted absolute timestamp or a relative offset"), caught by actually
|
||||
// running this against the live stack. Omitting the latest= clause
|
||||
// entirely is the query language's own way of saying "no upper bound",
|
||||
// which is exactly what "now" means here.
|
||||
export function injectTimeRange(query: string, earliest: string, latest: string): string {
|
||||
const clauses = [`earliest=${earliest}`];
|
||||
if (latest && latest !== 'now') clauses.push(`latest=${latest}`);
|
||||
return `${clauses.join(' ')} ${query}`;
|
||||
}
|
||||
|
||||
// --- alerting ---------------------------------------------------------
|
||||
|
||||
export type ConditionType = 'threshold' | 'absence';
|
||||
export type Comparator = 'gt' | 'gte' | 'lt' | 'lte' | 'eq' | 'ne';
|
||||
export type NotificationKind = 'webhook' | 'slack' | 'pagerduty';
|
||||
export type AlertRuleState = 'ok' | 'pending' | 'firing';
|
||||
|
||||
export type NotificationTarget = {
|
||||
id: string;
|
||||
name: string;
|
||||
kind: NotificationKind;
|
||||
webhook_url: string;
|
||||
};
|
||||
|
||||
export type AlertRule = {
|
||||
id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
query: string;
|
||||
query_language: Language;
|
||||
condition_type: ConditionType;
|
||||
comparator?: Comparator;
|
||||
threshold_value?: number;
|
||||
eval_interval_seconds: number;
|
||||
for_minutes: number;
|
||||
renotify_interval_minutes?: number;
|
||||
notification_target_id: string;
|
||||
enabled: boolean;
|
||||
state: {
|
||||
state: AlertRuleState;
|
||||
last_evaluated_at?: string;
|
||||
last_eval_status: 'ok' | 'error';
|
||||
last_error?: string;
|
||||
last_value?: number;
|
||||
consecutive_errors: number;
|
||||
};
|
||||
};
|
||||
|
||||
export type DeliveryLogEntry = {
|
||||
id: number;
|
||||
event_type: 'firing' | 'resolved';
|
||||
status: 'pending' | 'sent' | 'failed' | 'retrying';
|
||||
attempt_count: number;
|
||||
last_error?: string;
|
||||
response_status?: number;
|
||||
created_at: string;
|
||||
};
|
||||
|
||||
export function listRules(): Promise<AlertRule[]> {
|
||||
return alertingRequest<AlertRule[]>('/rules').then((r) => r ?? []);
|
||||
}
|
||||
|
||||
export function getRule(id: string): Promise<AlertRule> {
|
||||
return alertingRequest(`/rules/${id}`);
|
||||
}
|
||||
|
||||
export function createRule(input: Partial<AlertRule>): Promise<AlertRule> {
|
||||
return alertingRequest('/rules', { method: 'POST', body: JSON.stringify(input) });
|
||||
}
|
||||
|
||||
export function deleteRule(id: string): Promise<void> {
|
||||
return alertingRequest(`/rules/${id}`, { method: 'DELETE' });
|
||||
}
|
||||
|
||||
export function listDeliveries(ruleId: string): Promise<DeliveryLogEntry[]> {
|
||||
return alertingRequest<DeliveryLogEntry[]>(`/rules/${ruleId}/deliveries`).then((d) => d ?? []);
|
||||
}
|
||||
|
||||
export function listNotificationTargets(): Promise<NotificationTarget[]> {
|
||||
return alertingRequest<NotificationTarget[]>('/targets').then((t) => t ?? []);
|
||||
}
|
||||
|
||||
export function createNotificationTarget(input: {
|
||||
name: string;
|
||||
kind: NotificationKind;
|
||||
webhook_url: string;
|
||||
}): Promise<NotificationTarget> {
|
||||
return alertingRequest('/targets', { method: 'POST', body: JSON.stringify(input) });
|
||||
}
|
||||
Reference in New Issue
Block a user