Phase 2: unified query language spanning ClickHouse and Tantivy

Replaces the separate SQL-only /query and text-only /search endpoints
with one pipe-syntax query language (plus raw SQL escape hatch) that
compiles to a single IR and execution plan across both backends, so a
query like `message:"connection refused" | stats count by host` runs
as one request instead of two disjoint tools.

- api/internal/querylang: lexer -> ast -> parser -> ir -> planner ->
  executor, each layer independently tested.
- Execution generalizes Phase 1's proven Tantivy-prefilter pattern
  into a 4-way routing table (pure ClickHouse / text-only / text +
  aggregation / raw SQL passthrough).
- Unified web query page and `sentryctl query`, both hitting the same
  POST /query endpoint.
- Benchmarked against a real 1,022,000-row dataset
  (hack/benchmark-fixture); caught and fixed a real bug where the
  Tantivy prefilter cap (10,000) produced an IN-clause exceeding
  ClickHouse's default max_query_size -- lowered to 5,000, documented
  in docs/query-language-design.md and docs/phase-2-runbook.md.
- docs/query-language-reference.md: customer-facing syntax reference.
This commit is contained in:
2026-08-13 12:21:42 -07:00
parent cd8aa290ca
commit fb5049a747
36 changed files with 4119 additions and 613 deletions
-29
View File
@@ -1,6 +1,5 @@
<script lang="ts">
import favicon from '$lib/assets/favicon.svg';
import { page } from '$app/state';
let { children } = $props();
</script>
@@ -10,32 +9,4 @@
<link rel="icon" href={favicon} />
</svelte:head>
<nav>
<a href="/" class:active={page.url.pathname === '/'}>SQL Query</a>
<a href="/search" class:active={page.url.pathname === '/search'}>Full-Text Search</a>
</nav>
{@render children()}
<style>
nav {
font-family: system-ui, sans-serif;
max-width: 960px;
margin: 1rem auto 0;
padding: 0 1rem;
display: flex;
gap: 1rem;
border-bottom: 1px solid #ccc;
}
nav a {
padding: 0.5rem 0;
text-decoration: none;
color: #555;
border-bottom: 2px solid transparent;
}
nav a.active {
color: #000;
border-bottom-color: #000;
font-weight: 600;
}
</style>
+168 -14
View File
@@ -1,19 +1,61 @@
<script lang="ts">
// Phase 0: functional only, no styling polish, no auth. One page: a raw
// SQL box against POST /query on the api service, rendered as a table.
// This is a placeholder for the real query UI that lands once /api grows
// a real SPL-like query layer in Phase 2.
// Phase 2: single unified query page. Replaces Phase 0/1's two
// separate pages (raw-SQL-only /query, free-text-only /search) --
// see /docs/query-language-design.md and /docs/query-language-reference.md.
import ResultsTable from '$lib/ResultsTable.svelte';
const apiBase = import.meta.env.VITE_API_BASE_URL ?? 'http://localhost:8080';
let sql = $state('SELECT * FROM logs ORDER BY timestamp DESC LIMIT 100');
type Language = '' | 'sql' | 'spl';
type HistoryEntry = { query: string; language: Language; at: number };
const HISTORY_KEY = 'sentry.queryHistory';
const HISTORY_LIMIT = 20;
let query = $state('earliest=-1h | sort -timestamp | head 100');
let language = $state<Language>('');
let columns = $state<string[]>([]);
let rows = $state<unknown[][]>([]);
let error = $state('');
let loading = $state(false);
let hasRun = $state(false);
let history = $state<HistoryEntry[]>(loadHistory());
// Client-side mirror of the backend's auto-detect heuristic
// (api/internal/querylang/planner.looksLikeSQL) -- purely a UI hint,
// the server does its own detection independently and is the
// authority on what actually runs.
function detectedLanguage(q: string): 'sql' | 'spl' {
return /^\s*select\b/i.test(q) ? 'sql' : 'spl';
}
let detected = $derived(detectedLanguage(query));
let effectiveLanguage = $derived(language === '' ? detected : language);
function loadHistory(): HistoryEntry[] {
if (typeof sessionStorage === 'undefined') return [];
try {
const raw = sessionStorage.getItem(HISTORY_KEY);
return raw ? JSON.parse(raw) : [];
} catch {
return [];
}
}
function saveHistory(entry: HistoryEntry) {
history = [entry, ...history.filter((h) => h.query !== entry.query)].slice(0, HISTORY_LIMIT);
try {
sessionStorage.setItem(HISTORY_KEY, JSON.stringify(history));
} catch {
// session storage unavailable/full -- history is a convenience,
// not worth failing the query over
}
}
function useHistoryEntry(entry: HistoryEntry) {
query = entry.query;
language = entry.language;
}
async function runQuery() {
loading = true;
@@ -22,7 +64,7 @@
const res = await fetch(`${apiBase}/query`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ sql })
body: JSON.stringify({ query, language })
});
const body = await res.json();
if (!res.ok) {
@@ -33,6 +75,7 @@
}
columns = body.columns ?? [];
rows = body.rows ?? [];
saveHistory({ query, language, at: Date.now() });
} catch (e) {
error = e instanceof Error ? e.message : String(e);
columns = [];
@@ -42,21 +85,51 @@
hasRun = true;
}
}
function onKeydown(e: KeyboardEvent) {
// Cmd/Ctrl+Enter runs the query -- textarea's own Enter key needs
// to stay newline-for-pipe-stage-formatting, so this isn't a bare
// Enter binding.
if ((e.metaKey || e.ctrlKey) && e.key === 'Enter') {
e.preventDefault();
runQuery();
}
}
</script>
<main>
<h1>Sentry — Log Query</h1>
<h1>Sentry — Query</h1>
<p>
Raw SQL only, SELECT statements against the <code>logs</code> table. No auth, no query
builder yet — see <code>/api</code> for what's actually allowed. Looking for free-text
search instead? See the <a href="/search">Full-Text Search</a> page.
One query bar for both filter/stats queries and free-text search — see
<code>/docs/query-language-reference.md</code> in the repo for the full syntax, or the cheat
sheet below.
</p>
<textarea bind:value={sql} rows="4" cols="100" spellcheck="false"></textarea>
<div>
<button onclick={runQuery} disabled={loading}>
<textarea
bind:value={query}
onkeydown={onKeydown}
rows="4"
cols="100"
spellcheck="false"
placeholder={'service=api | where status>=500 | stats count by host | sort -count'}
></textarea>
<div class="controls">
<label>
Language:
<select bind:value={language}>
<option value="">Auto ({detected})</option>
<option value="spl">Pipe syntax</option>
<option value="sql">SQL</option>
</select>
</label>
<span class="detected-badge" class:sql={effectiveLanguage === 'sql'}>
{effectiveLanguage === 'sql' ? 'SQL' : 'pipe syntax'}
</span>
<button onclick={runQuery} disabled={loading || query.trim() === ''}>
{loading ? 'Running…' : 'Run query'}
</button>
<span class="hint">⌘/Ctrl+Enter to run</span>
</div>
{#if error}
@@ -64,6 +137,39 @@
{/if}
<ResultsTable {columns} {rows} {hasRun} />
{#if history.length > 0}
<details class="history">
<summary>Query history ({history.length})</summary>
<ul>
{#each history as entry (entry.at)}
<li>
<button class="history-item" onclick={() => useHistoryEntry(entry)}>
<code>{entry.query}</code>
</button>
</li>
{/each}
</ul>
</details>
{/if}
<details class="cheatsheet">
<summary>Pipe syntax cheat sheet</summary>
<table>
<tbody>
<tr><td><code>field=value</code></td><td>filter on a structured field</td></tr>
<tr><td><code>"free text"</code> / bare word</td><td>full-text search on <code>message</code></td></tr>
<tr><td><code>message:"exact phrase"</code></td><td>explicit full-text search</td></tr>
<tr><td><code>| where field&gt;value</code></td><td>additional structured filter</td></tr>
<tr><td><code>| stats count by field</code></td><td>aggregate (count/sum/avg/min/max)</td></tr>
<tr><td><code>| sort -field</code></td><td>sort descending (<code>+field</code> for ascending)</td></tr>
<tr><td><code>| fields a, b</code></td><td>project specific columns</td></tr>
<tr><td><code>| head 50</code> / <code>| tail 50</code></td><td>limit results</td></tr>
<tr><td><code>earliest=-1h</code> / <code>latest=...</code></td><td>time range (relative or RFC3339)</td></tr>
</tbody>
</table>
<p>Full reference: <code>/docs/query-language-reference.md</code> in the repo.</p>
</details>
</main>
<style>
@@ -77,11 +183,59 @@
width: 100%;
font-family: monospace;
font-size: 0.9rem;
box-sizing: border-box;
}
button {
.controls {
margin-top: 0.5rem;
display: flex;
align-items: center;
gap: 0.75rem;
flex-wrap: wrap;
}
.detected-badge {
font-size: 0.75rem;
padding: 0.15rem 0.5rem;
border-radius: 1rem;
background: #eef;
color: #224;
}
.detected-badge.sql {
background: #fee;
color: #422;
}
.hint {
font-size: 0.8rem;
color: #777;
}
.error {
color: #b00020;
}
.history ul {
list-style: none;
padding: 0;
margin: 0.5rem 0 0;
}
.history-item {
background: none;
border: none;
text-align: left;
padding: 0.25rem 0;
cursor: pointer;
color: #06c;
}
.history-item:hover {
text-decoration: underline;
}
.cheatsheet {
margin-top: 1.5rem;
font-size: 0.85rem;
}
.cheatsheet table {
border-collapse: collapse;
margin-top: 0.5rem;
}
.cheatsheet td {
padding: 0.2rem 0.75rem 0.2rem 0;
vertical-align: top;
}
</style>
-96
View File
@@ -1,96 +0,0 @@
<script lang="ts">
// Phase 1: free-text search via POST /search on the api service, hits
// the Tantivy-backed search service and returns full rows joined back
// against ClickHouse. No unified query experience with the SQL page —
// that's Phase 2's job.
import ResultsTable from '$lib/ResultsTable.svelte';
const apiBase = import.meta.env.VITE_API_BASE_URL ?? 'http://localhost:8080';
let query = $state('');
let columns = $state<string[]>([]);
let rows = $state<unknown[][]>([]);
let error = $state('');
let loading = $state(false);
let hasRun = $state(false);
async function runSearch() {
loading = true;
error = '';
try {
const res = await fetch(`${apiBase}/search`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ query })
});
const body = await res.json();
if (!res.ok) {
error = body?.error ?? `request failed with status ${res.status}`;
columns = [];
rows = [];
return;
}
columns = body.columns ?? [];
rows = body.rows ?? [];
} catch (e) {
error = e instanceof Error ? e.message : String(e);
columns = [];
rows = [];
} finally {
loading = false;
hasRun = true;
}
}
</script>
<main>
<h1>Sentry — Full-Text Search</h1>
<p>
Free-text search over the <code>message</code> field, via Tantivy. Supports plain terms,
<code>"exact phrases"</code>, and <code>wildcard*</code> — see
<code>/search</code> for the full query syntax. Looking for structured/aggregation queries
instead? See the <a href="/">SQL Query</a> page.
</p>
<input
type="text"
bind:value={query}
placeholder="e.g. &quot;connection refused&quot; or timeout*"
spellcheck="false"
onkeydown={(e) => e.key === 'Enter' && runSearch()}
/>
<div>
<button onclick={runSearch} disabled={loading || query.trim() === ''}>
{loading ? 'Searching…' : 'Search'}
</button>
</div>
{#if error}
<p class="error">Error: {error}</p>
{/if}
<ResultsTable {columns} {rows} {hasRun} />
</main>
<style>
main {
font-family: system-ui, sans-serif;
max-width: 960px;
margin: 2rem auto;
padding: 0 1rem;
}
input {
width: 100%;
font-family: monospace;
font-size: 0.9rem;
padding: 0.4rem;
box-sizing: border-box;
}
button {
margin-top: 0.5rem;
}
.error {
color: #b00020;
}
</style>
-4
View File
@@ -1,4 +0,0 @@
// Same reasoning as the root page's +page.ts: no load function (all data
// comes from a client-side fetch on submit), so a plain prerender is
// enough for the static adapter.
export const prerender = true;