Phase 2: unified query language spanning ClickHouse and Tantivy
Replaces the separate SQL-only /query and text-only /search endpoints with one pipe-syntax query language (plus raw SQL escape hatch) that compiles to a single IR and execution plan across both backends, so a query like `message:"connection refused" | stats count by host` runs as one request instead of two disjoint tools. - api/internal/querylang: lexer -> ast -> parser -> ir -> planner -> executor, each layer independently tested. - Execution generalizes Phase 1's proven Tantivy-prefilter pattern into a 4-way routing table (pure ClickHouse / text-only / text + aggregation / raw SQL passthrough). - Unified web query page and `sentryctl query`, both hitting the same POST /query endpoint. - Benchmarked against a real 1,022,000-row dataset (hack/benchmark-fixture); caught and fixed a real bug where the Tantivy prefilter cap (10,000) produced an IN-clause exceeding ClickHouse's default max_query_size -- lowered to 5,000, documented in docs/query-language-design.md and docs/phase-2-runbook.md. - docs/query-language-reference.md: customer-facing syntax reference.
This commit is contained in:
@@ -1,6 +1,5 @@
|
||||
<script lang="ts">
|
||||
import favicon from '$lib/assets/favicon.svg';
|
||||
import { page } from '$app/state';
|
||||
|
||||
let { children } = $props();
|
||||
</script>
|
||||
@@ -10,32 +9,4 @@
|
||||
<link rel="icon" href={favicon} />
|
||||
</svelte:head>
|
||||
|
||||
<nav>
|
||||
<a href="/" class:active={page.url.pathname === '/'}>SQL Query</a>
|
||||
<a href="/search" class:active={page.url.pathname === '/search'}>Full-Text Search</a>
|
||||
</nav>
|
||||
|
||||
{@render children()}
|
||||
|
||||
<style>
|
||||
nav {
|
||||
font-family: system-ui, sans-serif;
|
||||
max-width: 960px;
|
||||
margin: 1rem auto 0;
|
||||
padding: 0 1rem;
|
||||
display: flex;
|
||||
gap: 1rem;
|
||||
border-bottom: 1px solid #ccc;
|
||||
}
|
||||
nav a {
|
||||
padding: 0.5rem 0;
|
||||
text-decoration: none;
|
||||
color: #555;
|
||||
border-bottom: 2px solid transparent;
|
||||
}
|
||||
nav a.active {
|
||||
color: #000;
|
||||
border-bottom-color: #000;
|
||||
font-weight: 600;
|
||||
}
|
||||
</style>
|
||||
|
||||
+168
-14
@@ -1,19 +1,61 @@
|
||||
<script lang="ts">
|
||||
// Phase 0: functional only, no styling polish, no auth. One page: a raw
|
||||
// SQL box against POST /query on the api service, rendered as a table.
|
||||
// This is a placeholder for the real query UI that lands once /api grows
|
||||
// a real SPL-like query layer in Phase 2.
|
||||
// Phase 2: single unified query page. Replaces Phase 0/1's two
|
||||
// separate pages (raw-SQL-only /query, free-text-only /search) --
|
||||
// see /docs/query-language-design.md and /docs/query-language-reference.md.
|
||||
|
||||
import ResultsTable from '$lib/ResultsTable.svelte';
|
||||
|
||||
const apiBase = import.meta.env.VITE_API_BASE_URL ?? 'http://localhost:8080';
|
||||
|
||||
let sql = $state('SELECT * FROM logs ORDER BY timestamp DESC LIMIT 100');
|
||||
type Language = '' | 'sql' | 'spl';
|
||||
type HistoryEntry = { query: string; language: Language; at: number };
|
||||
|
||||
const HISTORY_KEY = 'sentry.queryHistory';
|
||||
const HISTORY_LIMIT = 20;
|
||||
|
||||
let query = $state('earliest=-1h | sort -timestamp | head 100');
|
||||
let language = $state<Language>('');
|
||||
let columns = $state<string[]>([]);
|
||||
let rows = $state<unknown[][]>([]);
|
||||
let error = $state('');
|
||||
let loading = $state(false);
|
||||
let hasRun = $state(false);
|
||||
let history = $state<HistoryEntry[]>(loadHistory());
|
||||
|
||||
// Client-side mirror of the backend's auto-detect heuristic
|
||||
// (api/internal/querylang/planner.looksLikeSQL) -- purely a UI hint,
|
||||
// the server does its own detection independently and is the
|
||||
// authority on what actually runs.
|
||||
function detectedLanguage(q: string): 'sql' | 'spl' {
|
||||
return /^\s*select\b/i.test(q) ? 'sql' : 'spl';
|
||||
}
|
||||
let detected = $derived(detectedLanguage(query));
|
||||
let effectiveLanguage = $derived(language === '' ? detected : language);
|
||||
|
||||
function loadHistory(): HistoryEntry[] {
|
||||
if (typeof sessionStorage === 'undefined') return [];
|
||||
try {
|
||||
const raw = sessionStorage.getItem(HISTORY_KEY);
|
||||
return raw ? JSON.parse(raw) : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function saveHistory(entry: HistoryEntry) {
|
||||
history = [entry, ...history.filter((h) => h.query !== entry.query)].slice(0, HISTORY_LIMIT);
|
||||
try {
|
||||
sessionStorage.setItem(HISTORY_KEY, JSON.stringify(history));
|
||||
} catch {
|
||||
// session storage unavailable/full -- history is a convenience,
|
||||
// not worth failing the query over
|
||||
}
|
||||
}
|
||||
|
||||
function useHistoryEntry(entry: HistoryEntry) {
|
||||
query = entry.query;
|
||||
language = entry.language;
|
||||
}
|
||||
|
||||
async function runQuery() {
|
||||
loading = true;
|
||||
@@ -22,7 +64,7 @@
|
||||
const res = await fetch(`${apiBase}/query`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ sql })
|
||||
body: JSON.stringify({ query, language })
|
||||
});
|
||||
const body = await res.json();
|
||||
if (!res.ok) {
|
||||
@@ -33,6 +75,7 @@
|
||||
}
|
||||
columns = body.columns ?? [];
|
||||
rows = body.rows ?? [];
|
||||
saveHistory({ query, language, at: Date.now() });
|
||||
} catch (e) {
|
||||
error = e instanceof Error ? e.message : String(e);
|
||||
columns = [];
|
||||
@@ -42,21 +85,51 @@
|
||||
hasRun = true;
|
||||
}
|
||||
}
|
||||
|
||||
function onKeydown(e: KeyboardEvent) {
|
||||
// Cmd/Ctrl+Enter runs the query -- textarea's own Enter key needs
|
||||
// to stay newline-for-pipe-stage-formatting, so this isn't a bare
|
||||
// Enter binding.
|
||||
if ((e.metaKey || e.ctrlKey) && e.key === 'Enter') {
|
||||
e.preventDefault();
|
||||
runQuery();
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<main>
|
||||
<h1>Sentry — Log Query</h1>
|
||||
<h1>Sentry — Query</h1>
|
||||
<p>
|
||||
Raw SQL only, SELECT statements against the <code>logs</code> table. No auth, no query
|
||||
builder yet — see <code>/api</code> for what's actually allowed. Looking for free-text
|
||||
search instead? See the <a href="/search">Full-Text Search</a> page.
|
||||
One query bar for both filter/stats queries and free-text search — see
|
||||
<code>/docs/query-language-reference.md</code> in the repo for the full syntax, or the cheat
|
||||
sheet below.
|
||||
</p>
|
||||
|
||||
<textarea bind:value={sql} rows="4" cols="100" spellcheck="false"></textarea>
|
||||
<div>
|
||||
<button onclick={runQuery} disabled={loading}>
|
||||
<textarea
|
||||
bind:value={query}
|
||||
onkeydown={onKeydown}
|
||||
rows="4"
|
||||
cols="100"
|
||||
spellcheck="false"
|
||||
placeholder={'service=api | where status>=500 | stats count by host | sort -count'}
|
||||
></textarea>
|
||||
|
||||
<div class="controls">
|
||||
<label>
|
||||
Language:
|
||||
<select bind:value={language}>
|
||||
<option value="">Auto ({detected})</option>
|
||||
<option value="spl">Pipe syntax</option>
|
||||
<option value="sql">SQL</option>
|
||||
</select>
|
||||
</label>
|
||||
<span class="detected-badge" class:sql={effectiveLanguage === 'sql'}>
|
||||
{effectiveLanguage === 'sql' ? 'SQL' : 'pipe syntax'}
|
||||
</span>
|
||||
<button onclick={runQuery} disabled={loading || query.trim() === ''}>
|
||||
{loading ? 'Running…' : 'Run query'}
|
||||
</button>
|
||||
<span class="hint">⌘/Ctrl+Enter to run</span>
|
||||
</div>
|
||||
|
||||
{#if error}
|
||||
@@ -64,6 +137,39 @@
|
||||
{/if}
|
||||
|
||||
<ResultsTable {columns} {rows} {hasRun} />
|
||||
|
||||
{#if history.length > 0}
|
||||
<details class="history">
|
||||
<summary>Query history ({history.length})</summary>
|
||||
<ul>
|
||||
{#each history as entry (entry.at)}
|
||||
<li>
|
||||
<button class="history-item" onclick={() => useHistoryEntry(entry)}>
|
||||
<code>{entry.query}</code>
|
||||
</button>
|
||||
</li>
|
||||
{/each}
|
||||
</ul>
|
||||
</details>
|
||||
{/if}
|
||||
|
||||
<details class="cheatsheet">
|
||||
<summary>Pipe syntax cheat sheet</summary>
|
||||
<table>
|
||||
<tbody>
|
||||
<tr><td><code>field=value</code></td><td>filter on a structured field</td></tr>
|
||||
<tr><td><code>"free text"</code> / bare word</td><td>full-text search on <code>message</code></td></tr>
|
||||
<tr><td><code>message:"exact phrase"</code></td><td>explicit full-text search</td></tr>
|
||||
<tr><td><code>| where field>value</code></td><td>additional structured filter</td></tr>
|
||||
<tr><td><code>| stats count by field</code></td><td>aggregate (count/sum/avg/min/max)</td></tr>
|
||||
<tr><td><code>| sort -field</code></td><td>sort descending (<code>+field</code> for ascending)</td></tr>
|
||||
<tr><td><code>| fields a, b</code></td><td>project specific columns</td></tr>
|
||||
<tr><td><code>| head 50</code> / <code>| tail 50</code></td><td>limit results</td></tr>
|
||||
<tr><td><code>earliest=-1h</code> / <code>latest=...</code></td><td>time range (relative or RFC3339)</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<p>Full reference: <code>/docs/query-language-reference.md</code> in the repo.</p>
|
||||
</details>
|
||||
</main>
|
||||
|
||||
<style>
|
||||
@@ -77,11 +183,59 @@
|
||||
width: 100%;
|
||||
font-family: monospace;
|
||||
font-size: 0.9rem;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
button {
|
||||
.controls {
|
||||
margin-top: 0.5rem;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.75rem;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
.detected-badge {
|
||||
font-size: 0.75rem;
|
||||
padding: 0.15rem 0.5rem;
|
||||
border-radius: 1rem;
|
||||
background: #eef;
|
||||
color: #224;
|
||||
}
|
||||
.detected-badge.sql {
|
||||
background: #fee;
|
||||
color: #422;
|
||||
}
|
||||
.hint {
|
||||
font-size: 0.8rem;
|
||||
color: #777;
|
||||
}
|
||||
.error {
|
||||
color: #b00020;
|
||||
}
|
||||
.history ul {
|
||||
list-style: none;
|
||||
padding: 0;
|
||||
margin: 0.5rem 0 0;
|
||||
}
|
||||
.history-item {
|
||||
background: none;
|
||||
border: none;
|
||||
text-align: left;
|
||||
padding: 0.25rem 0;
|
||||
cursor: pointer;
|
||||
color: #06c;
|
||||
}
|
||||
.history-item:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
.cheatsheet {
|
||||
margin-top: 1.5rem;
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
.cheatsheet table {
|
||||
border-collapse: collapse;
|
||||
margin-top: 0.5rem;
|
||||
}
|
||||
.cheatsheet td {
|
||||
padding: 0.2rem 0.75rem 0.2rem 0;
|
||||
vertical-align: top;
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
<script lang="ts">
|
||||
// Phase 1: free-text search via POST /search on the api service, hits
|
||||
// the Tantivy-backed search service and returns full rows joined back
|
||||
// against ClickHouse. No unified query experience with the SQL page —
|
||||
// that's Phase 2's job.
|
||||
|
||||
import ResultsTable from '$lib/ResultsTable.svelte';
|
||||
|
||||
const apiBase = import.meta.env.VITE_API_BASE_URL ?? 'http://localhost:8080';
|
||||
|
||||
let query = $state('');
|
||||
let columns = $state<string[]>([]);
|
||||
let rows = $state<unknown[][]>([]);
|
||||
let error = $state('');
|
||||
let loading = $state(false);
|
||||
let hasRun = $state(false);
|
||||
|
||||
async function runSearch() {
|
||||
loading = true;
|
||||
error = '';
|
||||
try {
|
||||
const res = await fetch(`${apiBase}/search`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ query })
|
||||
});
|
||||
const body = await res.json();
|
||||
if (!res.ok) {
|
||||
error = body?.error ?? `request failed with status ${res.status}`;
|
||||
columns = [];
|
||||
rows = [];
|
||||
return;
|
||||
}
|
||||
columns = body.columns ?? [];
|
||||
rows = body.rows ?? [];
|
||||
} catch (e) {
|
||||
error = e instanceof Error ? e.message : String(e);
|
||||
columns = [];
|
||||
rows = [];
|
||||
} finally {
|
||||
loading = false;
|
||||
hasRun = true;
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<main>
|
||||
<h1>Sentry — Full-Text Search</h1>
|
||||
<p>
|
||||
Free-text search over the <code>message</code> field, via Tantivy. Supports plain terms,
|
||||
<code>"exact phrases"</code>, and <code>wildcard*</code> — see
|
||||
<code>/search</code> for the full query syntax. Looking for structured/aggregation queries
|
||||
instead? See the <a href="/">SQL Query</a> page.
|
||||
</p>
|
||||
|
||||
<input
|
||||
type="text"
|
||||
bind:value={query}
|
||||
placeholder="e.g. "connection refused" or timeout*"
|
||||
spellcheck="false"
|
||||
onkeydown={(e) => e.key === 'Enter' && runSearch()}
|
||||
/>
|
||||
<div>
|
||||
<button onclick={runSearch} disabled={loading || query.trim() === ''}>
|
||||
{loading ? 'Searching…' : 'Search'}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{#if error}
|
||||
<p class="error">Error: {error}</p>
|
||||
{/if}
|
||||
|
||||
<ResultsTable {columns} {rows} {hasRun} />
|
||||
</main>
|
||||
|
||||
<style>
|
||||
main {
|
||||
font-family: system-ui, sans-serif;
|
||||
max-width: 960px;
|
||||
margin: 2rem auto;
|
||||
padding: 0 1rem;
|
||||
}
|
||||
input {
|
||||
width: 100%;
|
||||
font-family: monospace;
|
||||
font-size: 0.9rem;
|
||||
padding: 0.4rem;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
button {
|
||||
margin-top: 0.5rem;
|
||||
}
|
||||
.error {
|
||||
color: #b00020;
|
||||
}
|
||||
</style>
|
||||
@@ -1,4 +0,0 @@
|
||||
// Same reasoning as the root page's +page.ts: no load function (all data
|
||||
// comes from a client-side fetch on submit), so a plain prerender is
|
||||
// enough for the static adapter.
|
||||
export const prerender = true;
|
||||
Reference in New Issue
Block a user