Phase 2: unified query language spanning ClickHouse and Tantivy
Replaces the separate SQL-only /query and text-only /search endpoints with one pipe-syntax query language (plus raw SQL escape hatch) that compiles to a single IR and execution plan across both backends, so a query like `message:"connection refused" | stats count by host` runs as one request instead of two disjoint tools. - api/internal/querylang: lexer -> ast -> parser -> ir -> planner -> executor, each layer independently tested. - Execution generalizes Phase 1's proven Tantivy-prefilter pattern into a 4-way routing table (pure ClickHouse / text-only / text + aggregation / raw SQL passthrough). - Unified web query page and `sentryctl query`, both hitting the same POST /query endpoint. - Benchmarked against a real 1,022,000-row dataset (hack/benchmark-fixture); caught and fixed a real bug where the Tantivy prefilter cap (10,000) produced an IN-clause exceeding ClickHouse's default max_query_size -- lowered to 5,000, documented in docs/query-language-design.md and docs/phase-2-runbook.md. - docs/query-language-reference.md: customer-facing syntax reference.
This commit is contained in:
@@ -0,0 +1,80 @@
|
||||
// Package ir defines Plan, the intermediate representation both the
|
||||
// pipe-syntax parser+planner and the raw-SQL passthrough compile down
|
||||
// to. This is the boundary task 3 asked for: "pipe syntax X compiles to
|
||||
// IR Y" is testable in planner without any backend; "IR Y executes
|
||||
// correctly" is testable in executor against fakes, independent of the
|
||||
// planner. See /docs/query-language-design.md.
|
||||
package ir
|
||||
|
||||
import "time"
|
||||
|
||||
type Plan struct {
|
||||
// RawSQL, when non-empty, means the entire plan is this opaque
|
||||
// ClickHouse SQL string, executed as-is -- every other field below
|
||||
// is unused. This is the SQL escape hatch's IR representation: a
|
||||
// trivial identity compilation that still flows through the same
|
||||
// Plan type and the same executor code path as a parsed pipe query.
|
||||
RawSQL string
|
||||
|
||||
// TextSearch predicates route to Tantivy as a prefilter. Empty means
|
||||
// no Tantivy involvement at all -- pure ClickHouse.
|
||||
TextSearch []TextPredicate
|
||||
|
||||
// Filters are always evaluated in ClickHouse, either directly as
|
||||
// WHERE clauses (no TextSearch present) or as an additional filter
|
||||
// alongside a Tantivy-sourced record_id IN (...) clause.
|
||||
Filters []FilterPredicate
|
||||
|
||||
TimeRange *TimeRange
|
||||
|
||||
// Aggregation is nil for a raw-rows query (no GROUP BY).
|
||||
Aggregation *Aggregation
|
||||
|
||||
Sort []SortField
|
||||
|
||||
// Fields is the projection; empty means all columns.
|
||||
Fields []string
|
||||
|
||||
Limit *Limit
|
||||
}
|
||||
|
||||
type TextPredicate struct {
|
||||
// Query is passed to Tantivy's query parser as-is -- phrase and
|
||||
// wildcard syntax already supported there (see /search).
|
||||
Query string
|
||||
}
|
||||
|
||||
type FilterPredicate struct {
|
||||
Field string
|
||||
Op string // "=", "!=", ">", ">=", "<", "<="
|
||||
Value string
|
||||
}
|
||||
|
||||
type Aggregation struct {
|
||||
Funcs []AggFunc
|
||||
GroupBy []string
|
||||
}
|
||||
|
||||
type AggFunc struct {
|
||||
Func string // count, sum, avg, min, max
|
||||
Field string // empty for count
|
||||
Alias string // always set by the planner (defaulted if not given explicitly)
|
||||
}
|
||||
|
||||
type SortField struct {
|
||||
Field string
|
||||
Desc bool
|
||||
}
|
||||
|
||||
type Limit struct {
|
||||
N int
|
||||
Tail bool // true = last N (by time), false = first N
|
||||
}
|
||||
|
||||
type TimeRange struct {
|
||||
// Absolute bounds -- any relative expression (-1h etc.) is resolved
|
||||
// by the planner at compile time, since only it knows "now". A zero
|
||||
// time.Time means that bound is unset.
|
||||
From time.Time
|
||||
To time.Time
|
||||
}
|
||||
Reference in New Issue
Block a user