Phase 2: unified query language spanning ClickHouse and Tantivy
Replaces the separate SQL-only /query and text-only /search endpoints with one pipe-syntax query language (plus raw SQL escape hatch) that compiles to a single IR and execution plan across both backends, so a query like `message:"connection refused" | stats count by host` runs as one request instead of two disjoint tools. - api/internal/querylang: lexer -> ast -> parser -> ir -> planner -> executor, each layer independently tested. - Execution generalizes Phase 1's proven Tantivy-prefilter pattern into a 4-way routing table (pure ClickHouse / text-only / text + aggregation / raw SQL passthrough). - Unified web query page and `sentryctl query`, both hitting the same POST /query endpoint. - Benchmarked against a real 1,022,000-row dataset (hack/benchmark-fixture); caught and fixed a real bug where the Tantivy prefilter cap (10,000) produced an IN-clause exceeding ClickHouse's default max_query_size -- lowered to 5,000, documented in docs/query-language-design.md and docs/phase-2-runbook.md. - docs/query-language-reference.md: customer-facing syntax reference.
This commit is contained in:
@@ -0,0 +1,58 @@
|
||||
package executor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
)
|
||||
|
||||
// ChRunner runs arbitrary (pre-validated) SELECT statements against
|
||||
// ClickHouse and shapes the result into JSON-friendly columns/rows,
|
||||
// discovering the result's column set at query time via reflection since
|
||||
// the query itself is arbitrary. Ported from Phase 0/1's
|
||||
// api/internal/queryapi.Executor, which this replaces (see task 4) --
|
||||
// same logic, moved here since it's the query-execution layer's
|
||||
// plumbing, not specific to the old placeholder /query handler.
|
||||
type ChRunner struct {
|
||||
conn driver.Conn
|
||||
}
|
||||
|
||||
func NewChRunner(conn driver.Conn) *ChRunner {
|
||||
return &ChRunner{conn: conn}
|
||||
}
|
||||
|
||||
func (r *ChRunner) RunSQL(ctx context.Context, sql string) (*Result, error) {
|
||||
rows, err := r.conn.Query(ctx, sql)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("executing query: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
columnTypes := rows.ColumnTypes()
|
||||
result := &Result{
|
||||
Columns: rows.Columns(),
|
||||
Rows: [][]any{},
|
||||
}
|
||||
|
||||
for rows.Next() {
|
||||
dest := make([]any, len(columnTypes))
|
||||
for i, ct := range columnTypes {
|
||||
dest[i] = reflect.New(ct.ScanType()).Interface()
|
||||
}
|
||||
if err := rows.Scan(dest...); err != nil {
|
||||
return nil, fmt.Errorf("scanning row: %w", err)
|
||||
}
|
||||
row := make([]any, len(dest))
|
||||
for i, d := range dest {
|
||||
row[i] = reflect.ValueOf(d).Elem().Interface()
|
||||
}
|
||||
result.Rows = append(result.Rows, row)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterating rows: %w", err)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
// Package executor runs a compiled ir.Plan and returns results in a
|
||||
// shape consistent regardless of which backend(s) were hit -- the point
|
||||
// of compiling to one IR in the first place. See
|
||||
// /docs/query-language-design.md's "Execution" section for the four
|
||||
// routing cases implemented here.
|
||||
package executor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/sentry/sentry/api/internal/querylang/ir"
|
||||
)
|
||||
|
||||
type Result struct {
|
||||
Columns []string
|
||||
Rows [][]any
|
||||
}
|
||||
|
||||
// SQLRunner executes a raw SQL statement against ClickHouse. *ChRunner
|
||||
// (chrunner.go) is the production implementation; tests use a fake --
|
||||
// same narrow-interface pattern used throughout /ingest and /api.
|
||||
type SQLRunner interface {
|
||||
RunSQL(ctx context.Context, sql string) (*Result, error)
|
||||
}
|
||||
|
||||
// SearchClient resolves a Tantivy query into matching record_ids.
|
||||
type SearchClient interface {
|
||||
Search(ctx context.Context, query string, limit uint32) ([]string, error)
|
||||
}
|
||||
|
||||
// textSearchLimit caps how many record_ids a Tantivy prefilter can feed
|
||||
// into a ClickHouse `IN (...)` clause. See /docs/query-language-design.md's
|
||||
// "Known scaling limitation" -- this is a real, disclosed limit on result
|
||||
// completeness for very broad text searches, not an oversight.
|
||||
//
|
||||
// 5000, not 10000: confirmed by actually running the Phase 2 benchmark
|
||||
// (see /docs/phase-2-runbook.md) that 10000 quoted UUIDs (~39 bytes each
|
||||
// including the comma) produces a ~390KB query string, which exceeds
|
||||
// ClickHouse's default max_query_size (262144 bytes / 256KiB) and fails
|
||||
// outright with a syntax error rather than degrading gracefully. 5000
|
||||
// UUIDs is ~195KB, safely under that default with headroom for the rest
|
||||
// of the query. This was a real failure caught by running the benchmark,
|
||||
// not a value chosen from first-principles estimation.
|
||||
const textSearchLimit = 5000
|
||||
|
||||
// Execute runs plan against the given backends. The four cases (per the
|
||||
// design doc): RawSQL passthrough; pure ClickHouse (no TextSearch); text
|
||||
// search alone (Tantivy prefilter -> ClickHouse row fetch); text search
|
||||
// plus aggregation (Tantivy prefilter -> ClickHouse aggregate). Cases 2-4
|
||||
// share the same buildSQL/buildWhereClause code (sql.go) -- the only
|
||||
// difference is whether a record_id filter is threaded in.
|
||||
func Execute(ctx context.Context, plan *ir.Plan, sqlRunner SQLRunner, search SearchClient) (*Result, error) {
|
||||
if plan.RawSQL != "" {
|
||||
return sqlRunner.RunSQL(ctx, plan.RawSQL)
|
||||
}
|
||||
|
||||
var recordIDFilter []string
|
||||
if len(plan.TextSearch) > 0 {
|
||||
ids, err := search.Search(ctx, plan.TextSearch[0].Query, textSearchLimit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("full-text search failed: %w", err)
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return &Result{Columns: []string{}, Rows: [][]any{}}, nil
|
||||
}
|
||||
recordIDFilter = ids
|
||||
}
|
||||
|
||||
sql := buildSQL(plan, recordIDFilter)
|
||||
return sqlRunner.RunSQL(ctx, sql)
|
||||
}
|
||||
@@ -0,0 +1,309 @@
|
||||
package executor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/sentry/sentry/api/internal/querylang/ir"
|
||||
)
|
||||
|
||||
func mustParseTime(t *testing.T, s string) time.Time {
|
||||
t.Helper()
|
||||
tm, err := time.Parse(time.RFC3339, s)
|
||||
if err != nil {
|
||||
t.Fatalf("parsing time %q: %v", s, err)
|
||||
}
|
||||
return tm
|
||||
}
|
||||
|
||||
type fakeSQLRunner struct {
|
||||
gotSQL string
|
||||
result *Result
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *fakeSQLRunner) RunSQL(_ context.Context, sql string) (*Result, error) {
|
||||
f.gotSQL = sql
|
||||
f.calls++
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
if f.result != nil {
|
||||
return f.result, nil
|
||||
}
|
||||
return &Result{Columns: []string{}, Rows: [][]any{}}, nil
|
||||
}
|
||||
|
||||
type fakeSearchClient struct {
|
||||
gotQuery string
|
||||
gotLimit uint32
|
||||
ids []string
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *fakeSearchClient) Search(_ context.Context, query string, limit uint32) ([]string, error) {
|
||||
f.gotQuery = query
|
||||
f.gotLimit = limit
|
||||
f.calls++
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
return f.ids, nil
|
||||
}
|
||||
|
||||
func TestExecuteRawSQLBypassesEverythingElse(t *testing.T) {
|
||||
sqlRunner := &fakeSQLRunner{}
|
||||
search := &fakeSearchClient{}
|
||||
plan := &ir.Plan{RawSQL: "SELECT 1"}
|
||||
|
||||
_, err := Execute(context.Background(), plan, sqlRunner, search)
|
||||
if err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
if sqlRunner.gotSQL != "SELECT 1" {
|
||||
t.Fatalf("gotSQL = %q, want %q", sqlRunner.gotSQL, "SELECT 1")
|
||||
}
|
||||
if search.calls != 0 {
|
||||
t.Fatalf("expected search not to be called for RawSQL, got %d calls", search.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutePureClickHousePathSkipsSearch(t *testing.T) {
|
||||
sqlRunner := &fakeSQLRunner{}
|
||||
search := &fakeSearchClient{}
|
||||
plan := &ir.Plan{Filters: []ir.FilterPredicate{{Field: "service", Op: "=", Value: "api"}}}
|
||||
|
||||
_, err := Execute(context.Background(), plan, sqlRunner, search)
|
||||
if err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
if search.calls != 0 {
|
||||
t.Fatalf("expected no search calls, got %d", search.calls)
|
||||
}
|
||||
if !strings.Contains(sqlRunner.gotSQL, "FROM logs") || !strings.Contains(sqlRunner.gotSQL, "`service` = 'api'") {
|
||||
t.Fatalf("unexpected SQL: %s", sqlRunner.gotSQL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteTextSearchPrefiltersThenQueriesClickHouse(t *testing.T) {
|
||||
sqlRunner := &fakeSQLRunner{}
|
||||
search := &fakeSearchClient{ids: []string{"id-1", "id-2"}}
|
||||
plan := &ir.Plan{TextSearch: []ir.TextPredicate{{Query: "connection refused"}}}
|
||||
|
||||
_, err := Execute(context.Background(), plan, sqlRunner, search)
|
||||
if err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
if search.gotQuery != "connection refused" {
|
||||
t.Fatalf("search query = %q", search.gotQuery)
|
||||
}
|
||||
if search.gotLimit != textSearchLimit {
|
||||
t.Fatalf("search limit = %d, want %d", search.gotLimit, textSearchLimit)
|
||||
}
|
||||
if !strings.Contains(sqlRunner.gotSQL, "record_id IN ('id-1','id-2')") {
|
||||
t.Fatalf("unexpected SQL: %s", sqlRunner.gotSQL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteTextSearchNoMatchesSkipsClickHouseEntirely(t *testing.T) {
|
||||
sqlRunner := &fakeSQLRunner{}
|
||||
search := &fakeSearchClient{ids: nil}
|
||||
plan := &ir.Plan{TextSearch: []ir.TextPredicate{{Query: "nothing matches"}}}
|
||||
|
||||
result, err := Execute(context.Background(), plan, sqlRunner, search)
|
||||
if err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
if sqlRunner.calls != 0 {
|
||||
t.Fatalf("expected ClickHouse not to be queried when search finds nothing, got %d calls", sqlRunner.calls)
|
||||
}
|
||||
if len(result.Columns) != 0 || len(result.Rows) != 0 {
|
||||
t.Fatalf("expected empty result, got %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteTextSearchWithAggregation(t *testing.T) {
|
||||
sqlRunner := &fakeSQLRunner{}
|
||||
search := &fakeSearchClient{ids: []string{"id-1"}}
|
||||
plan := &ir.Plan{
|
||||
TextSearch: []ir.TextPredicate{{Query: "connection refused"}},
|
||||
Aggregation: &ir.Aggregation{
|
||||
Funcs: []ir.AggFunc{{Func: "count", Alias: "count"}},
|
||||
GroupBy: []string{"host"},
|
||||
},
|
||||
}
|
||||
|
||||
_, err := Execute(context.Background(), plan, sqlRunner, search)
|
||||
if err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(sqlRunner.gotSQL, "record_id IN ('id-1')") {
|
||||
t.Fatalf("expected the text-search prefilter in the WHERE clause: %s", sqlRunner.gotSQL)
|
||||
}
|
||||
if !strings.Contains(sqlRunner.gotSQL, "GROUP BY `host`") {
|
||||
t.Fatalf("expected GROUP BY: %s", sqlRunner.gotSQL)
|
||||
}
|
||||
if !strings.Contains(sqlRunner.gotSQL, "count() AS `count`") {
|
||||
t.Fatalf("expected count() AS `count`: %s", sqlRunner.gotSQL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteSearchErrorPropagates(t *testing.T) {
|
||||
sqlRunner := &fakeSQLRunner{}
|
||||
search := &fakeSearchClient{err: errors.New("search unavailable")}
|
||||
plan := &ir.Plan{TextSearch: []ir.TextPredicate{{Query: "x"}}}
|
||||
|
||||
_, err := Execute(context.Background(), plan, sqlRunner, search)
|
||||
if err == nil {
|
||||
t.Fatal("expected the search error to propagate")
|
||||
}
|
||||
if sqlRunner.calls != 0 {
|
||||
t.Fatalf("expected ClickHouse not to be queried after a search error, got %d calls", sqlRunner.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSQLNumericCastOnAttributesField(t *testing.T) {
|
||||
plan := &ir.Plan{Filters: []ir.FilterPredicate{{Field: "status", Op: ">=", Value: "500"}}}
|
||||
sql := buildSQL(plan, nil)
|
||||
want := "toFloat64OrZero(attributes['status']) >= 500"
|
||||
if !strings.Contains(sql, want) {
|
||||
t.Fatalf("SQL = %q, want it to contain %q", sql, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSQLStringComparisonOnAttributesField(t *testing.T) {
|
||||
plan := &ir.Plan{Filters: []ir.FilterPredicate{{Field: "status", Op: "=", Value: "unknown"}}}
|
||||
sql := buildSQL(plan, nil)
|
||||
want := "attributes['status'] = 'unknown'"
|
||||
if !strings.Contains(sql, want) {
|
||||
t.Fatalf("SQL = %q, want it to contain %q", sql, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSQLTopLevelFieldNeverCast(t *testing.T) {
|
||||
plan := &ir.Plan{Filters: []ir.FilterPredicate{{Field: "service", Op: "=", Value: "123"}}}
|
||||
sql := buildSQL(plan, nil)
|
||||
if strings.Contains(sql, "toFloat64OrZero") {
|
||||
t.Fatalf("top-level field should never be numeric-cast: %s", sql)
|
||||
}
|
||||
if !strings.Contains(sql, "`service` = '123'") {
|
||||
t.Fatalf("unexpected SQL: %s", sql)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSQLEscapesInjectionAttemptInValue(t *testing.T) {
|
||||
plan := &ir.Plan{Filters: []ir.FilterPredicate{{Field: "service", Op: "=", Value: "x'; DROP TABLE logs; --"}}}
|
||||
sql := buildSQL(plan, nil)
|
||||
// The whole attacker-controlled value must land inside exactly one
|
||||
// quoted literal, with its embedded quote backslash-escaped so it
|
||||
// can't terminate the literal early -- checking for the escaped
|
||||
// form directly, not just the absence of the raw substring (which
|
||||
// is a weaker check: "\\'; DROP TABLE" still *contains* "'; DROP
|
||||
// TABLE" as a substring, so that alone doesn't prove escaping
|
||||
// happened).
|
||||
want := `'x\'; DROP TABLE logs; --'`
|
||||
if !strings.Contains(sql, want) {
|
||||
t.Fatalf("expected the literal %q in SQL, got: %s", want, sql)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSQLDefaultLimitAppliedWhenNoneGiven(t *testing.T) {
|
||||
plan := &ir.Plan{Filters: []ir.FilterPredicate{{Field: "service", Op: "=", Value: "api"}}}
|
||||
sql := buildSQL(plan, nil)
|
||||
if !strings.Contains(sql, "LIMIT 100") {
|
||||
t.Fatalf("expected the default row limit, got: %s", sql)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSQLExplicitLimitOverridesDefault(t *testing.T) {
|
||||
plan := &ir.Plan{
|
||||
Filters: []ir.FilterPredicate{{Field: "service", Op: "=", Value: "api"}},
|
||||
Limit: &ir.Limit{N: 5},
|
||||
}
|
||||
sql := buildSQL(plan, nil)
|
||||
if !strings.Contains(sql, "LIMIT 5") || strings.Contains(sql, "LIMIT 100") {
|
||||
t.Fatalf("expected LIMIT 5, got: %s", sql)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSQLTailWithoutSortOrdersAscending(t *testing.T) {
|
||||
plan := &ir.Plan{Limit: &ir.Limit{N: 10, Tail: true}}
|
||||
sql := buildSQL(plan, nil)
|
||||
if !strings.Contains(sql, "ORDER BY `timestamp` ASC") {
|
||||
t.Fatalf("expected ascending order for tail, got: %s", sql)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSQLNoSortDefaultsNewestFirst(t *testing.T) {
|
||||
plan := &ir.Plan{}
|
||||
sql := buildSQL(plan, nil)
|
||||
if !strings.Contains(sql, "ORDER BY `timestamp` DESC") {
|
||||
t.Fatalf("expected newest-first default, got: %s", sql)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSQLSortByAggregateAlias(t *testing.T) {
|
||||
plan := &ir.Plan{
|
||||
Aggregation: &ir.Aggregation{
|
||||
Funcs: []ir.AggFunc{{Func: "count", Alias: "count"}},
|
||||
GroupBy: []string{"host"},
|
||||
},
|
||||
Sort: []ir.SortField{{Field: "count", Desc: true}},
|
||||
}
|
||||
sql := buildSQL(plan, nil)
|
||||
if !strings.Contains(sql, "ORDER BY `count` DESC") {
|
||||
t.Fatalf("expected ORDER BY on the aggregate alias, got: %s", sql)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSQLSortByGroupByField(t *testing.T) {
|
||||
plan := &ir.Plan{
|
||||
Aggregation: &ir.Aggregation{
|
||||
Funcs: []ir.AggFunc{{Func: "count", Alias: "count"}},
|
||||
GroupBy: []string{"host"},
|
||||
},
|
||||
Sort: []ir.SortField{{Field: "host", Desc: false}},
|
||||
}
|
||||
sql := buildSQL(plan, nil)
|
||||
if !strings.Contains(sql, "ORDER BY `host` ASC") {
|
||||
t.Fatalf("expected ORDER BY on the group-by column, got: %s", sql)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSQLAggregationOnAttributesFieldAlwaysCasts(t *testing.T) {
|
||||
plan := &ir.Plan{
|
||||
Aggregation: &ir.Aggregation{
|
||||
Funcs: []ir.AggFunc{{Func: "avg", Field: "latency_ms", Alias: "avg_latency"}},
|
||||
},
|
||||
}
|
||||
sql := buildSQL(plan, nil)
|
||||
if !strings.Contains(sql, "AVG(toFloat64OrZero(attributes['latency_ms'])) AS `avg_latency`") {
|
||||
t.Fatalf("unexpected SQL: %s", sql)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSQLProjectionFields(t *testing.T) {
|
||||
plan := &ir.Plan{Fields: []string{"host", "message"}}
|
||||
sql := buildSQL(plan, nil)
|
||||
if !strings.Contains(sql, "SELECT `host` AS `host`, `message` AS `message` FROM logs") {
|
||||
t.Fatalf("unexpected SQL: %s", sql)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSQLTimeRange(t *testing.T) {
|
||||
from := mustParseTime(t, "2026-08-14T00:00:00Z")
|
||||
to := mustParseTime(t, "2026-08-14T01:00:00Z")
|
||||
plan := &ir.Plan{TimeRange: &ir.TimeRange{From: from, To: to}}
|
||||
sql := buildSQL(plan, nil)
|
||||
if !strings.Contains(sql, "`timestamp` >= '2026-08-14T00:00:00Z'") {
|
||||
t.Fatalf("missing From bound: %s", sql)
|
||||
}
|
||||
if !strings.Contains(sql, "`timestamp` <= '2026-08-14T01:00:00Z'") {
|
||||
t.Fatalf("missing To bound: %s", sql)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
package executor
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/sentry/sentry/api/internal/querylang/ir"
|
||||
)
|
||||
|
||||
// defaultRowLimit is the safety net when a raw-row query has neither an
|
||||
// explicit head/tail nor an aggregation -- without it, a bare `service=api`
|
||||
// with no other pipe stages would return every matching row unbounded.
|
||||
// Independent of planner's own defaultLimit (same value, different
|
||||
// concern: that one fills in `head`/`tail` with no N given; this one
|
||||
// guards queries that never mention head/tail at all).
|
||||
const defaultRowLimit = 100
|
||||
|
||||
// logs' real columns, per /storage. Anything else maps to
|
||||
// attributes['field'] -- see /docs/query-language-design.md's "Field
|
||||
// mapping" section.
|
||||
var topLevelFields = map[string]bool{
|
||||
"timestamp": true,
|
||||
"host": true,
|
||||
"service": true,
|
||||
"severity": true,
|
||||
"message": true,
|
||||
"record_id": true,
|
||||
}
|
||||
|
||||
func buildSQL(plan *ir.Plan, recordIDFilter []string) string {
|
||||
var sb strings.Builder
|
||||
|
||||
sb.WriteString("SELECT ")
|
||||
sb.WriteString(selectClause(plan))
|
||||
sb.WriteString(" FROM logs")
|
||||
|
||||
if where := buildWhereClause(plan, recordIDFilter); where != "" {
|
||||
sb.WriteString(" WHERE ")
|
||||
sb.WriteString(where)
|
||||
}
|
||||
|
||||
if plan.Aggregation != nil && len(plan.Aggregation.GroupBy) > 0 {
|
||||
sb.WriteString(" GROUP BY ")
|
||||
cols := make([]string, len(plan.Aggregation.GroupBy))
|
||||
for i, g := range plan.Aggregation.GroupBy {
|
||||
cols[i] = columnExpr(g)
|
||||
}
|
||||
sb.WriteString(strings.Join(cols, ", "))
|
||||
}
|
||||
|
||||
writeOrderBy(&sb, plan)
|
||||
|
||||
if plan.Limit != nil {
|
||||
fmt.Fprintf(&sb, " LIMIT %d", plan.Limit.N)
|
||||
} else if plan.Aggregation == nil {
|
||||
fmt.Fprintf(&sb, " LIMIT %d", defaultRowLimit)
|
||||
}
|
||||
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func writeOrderBy(sb *strings.Builder, plan *ir.Plan) {
|
||||
switch {
|
||||
case len(plan.Sort) > 0:
|
||||
sb.WriteString(" ORDER BY ")
|
||||
parts := make([]string, len(plan.Sort))
|
||||
for i, s := range plan.Sort {
|
||||
dir := "ASC"
|
||||
if s.Desc {
|
||||
dir = "DESC"
|
||||
}
|
||||
parts[i] = sortColumnExpr(plan, s.Field) + " " + dir
|
||||
}
|
||||
sb.WriteString(strings.Join(parts, ", "))
|
||||
case plan.Limit != nil && plan.Limit.Tail:
|
||||
// `tail N` with no explicit sort: order ascending so LIMIT N
|
||||
// takes the chronologically *last* N rows. Callers wanting
|
||||
// strict newest-first display order re-sort client-side --
|
||||
// documented in the query language reference.
|
||||
sb.WriteString(" ORDER BY `timestamp` ASC")
|
||||
case plan.Aggregation == nil:
|
||||
// Raw-row queries with no explicit sort default to newest-first,
|
||||
// matching the Phase 0/1 UI default.
|
||||
sb.WriteString(" ORDER BY `timestamp` DESC")
|
||||
}
|
||||
}
|
||||
|
||||
// sortColumnExpr resolves a sort field against an aggregation's own
|
||||
// output columns (alias or group-by field) before falling back to the
|
||||
// normal top-level/attributes mapping -- `sort -count` after `stats
|
||||
// count` refers to the aggregate's alias, not a raw column.
|
||||
func sortColumnExpr(plan *ir.Plan, field string) string {
|
||||
if plan.Aggregation != nil {
|
||||
for _, f := range plan.Aggregation.Funcs {
|
||||
if f.Alias == field {
|
||||
return quoteIdent(field)
|
||||
}
|
||||
}
|
||||
for _, g := range plan.Aggregation.GroupBy {
|
||||
if g == field {
|
||||
return columnExpr(field)
|
||||
}
|
||||
}
|
||||
}
|
||||
return columnExpr(field)
|
||||
}
|
||||
|
||||
func selectClause(plan *ir.Plan) string {
|
||||
if plan.Aggregation != nil {
|
||||
parts := make([]string, 0, len(plan.Aggregation.GroupBy)+len(plan.Aggregation.Funcs))
|
||||
for _, g := range plan.Aggregation.GroupBy {
|
||||
parts = append(parts, columnExpr(g)+" AS "+quoteIdent(g))
|
||||
}
|
||||
for _, f := range plan.Aggregation.Funcs {
|
||||
parts = append(parts, aggExpr(f)+" AS "+quoteIdent(f.Alias))
|
||||
}
|
||||
return strings.Join(parts, ", ")
|
||||
}
|
||||
if len(plan.Fields) > 0 {
|
||||
parts := make([]string, len(plan.Fields))
|
||||
for i, f := range plan.Fields {
|
||||
parts[i] = columnExpr(f) + " AS " + quoteIdent(f)
|
||||
}
|
||||
return strings.Join(parts, ", ")
|
||||
}
|
||||
return "*"
|
||||
}
|
||||
|
||||
// aggExpr always numeric-casts non-top-level (attributes-map) fields for
|
||||
// sum/avg/min/max, unlike comparison predicates where casting is
|
||||
// conditional on whether the compared value looks numeric -- an
|
||||
// aggregate function is inherently a numeric (or, for min/max,
|
||||
// order-comparable) operation, so there's no "maybe string" case the way
|
||||
// there is for `field=value`. Known Phase 2 limitation: min/max on a
|
||||
// non-top-level field always compares numerically, not lexicographically
|
||||
// -- string min/max on attributes isn't supported this phase.
|
||||
func aggExpr(f ir.AggFunc) string {
|
||||
if f.Func == "count" {
|
||||
return "count()"
|
||||
}
|
||||
col := columnExpr(f.Field)
|
||||
if !topLevelFields[f.Field] {
|
||||
col = "toFloat64OrZero(" + col + ")"
|
||||
}
|
||||
return strings.ToUpper(f.Func) + "(" + col + ")"
|
||||
}
|
||||
|
||||
func buildWhereClause(plan *ir.Plan, recordIDFilter []string) string {
|
||||
var conds []string
|
||||
|
||||
if len(recordIDFilter) > 0 {
|
||||
quoted := make([]string, len(recordIDFilter))
|
||||
for i, id := range recordIDFilter {
|
||||
quoted[i] = quoteLiteral(id)
|
||||
}
|
||||
conds = append(conds, "record_id IN ("+strings.Join(quoted, ",")+")")
|
||||
}
|
||||
|
||||
for _, f := range plan.Filters {
|
||||
conds = append(conds, buildComparisonSQL(f))
|
||||
}
|
||||
|
||||
if plan.TimeRange != nil {
|
||||
if !plan.TimeRange.From.IsZero() {
|
||||
conds = append(conds, "`timestamp` >= "+quoteLiteral(plan.TimeRange.From.UTC().Format(time.RFC3339Nano)))
|
||||
}
|
||||
if !plan.TimeRange.To.IsZero() {
|
||||
conds = append(conds, "`timestamp` <= "+quoteLiteral(plan.TimeRange.To.UTC().Format(time.RFC3339Nano)))
|
||||
}
|
||||
}
|
||||
|
||||
return strings.Join(conds, " AND ")
|
||||
}
|
||||
|
||||
// buildComparisonSQL numeric-casts a non-top-level field only when the
|
||||
// compared value itself looks numeric -- `status>=500` casts (numeric
|
||||
// comparison intent), `status="unknown"` doesn't (string comparison
|
||||
// intent). Top-level fields are never cast; ClickHouse compares them
|
||||
// against a string literal natively (DateTime64 columns parse an
|
||||
// RFC3339-shaped literal, LowCardinality(String)/String compare as-is).
|
||||
func buildComparisonSQL(f ir.FilterPredicate) string {
|
||||
if !topLevelFields[f.Field] && isNumericLiteral(f.Value) {
|
||||
return "toFloat64OrZero(" + columnExpr(f.Field) + ") " + f.Op + " " + f.Value
|
||||
}
|
||||
return columnExpr(f.Field) + " " + f.Op + " " + quoteLiteral(f.Value)
|
||||
}
|
||||
|
||||
func columnExpr(field string) string {
|
||||
if topLevelFields[field] {
|
||||
return quoteIdent(field)
|
||||
}
|
||||
return "attributes[" + quoteLiteral(field) + "]"
|
||||
}
|
||||
|
||||
func quoteIdent(name string) string {
|
||||
return "`" + strings.ReplaceAll(name, "`", "``") + "`"
|
||||
}
|
||||
|
||||
// quoteLiteral is the actual injection defense for every user-controlled
|
||||
// string embedded in generated SQL (filter values, attribute keys, time
|
||||
// bounds, record_ids). Field/keyword tokens from the lexer are already
|
||||
// constrained to [a-zA-Z0-9_.] by construction (see lexer.isIdentPart)
|
||||
// and can't carry SQL metacharacters at all, but quoted-string *values*
|
||||
// can contain anything, so this can't be skipped for them.
|
||||
func quoteLiteral(s string) string {
|
||||
var sb strings.Builder
|
||||
sb.WriteByte('\'')
|
||||
for _, r := range s {
|
||||
switch r {
|
||||
case '\\':
|
||||
sb.WriteString(`\\`)
|
||||
case '\'':
|
||||
sb.WriteString(`\'`)
|
||||
default:
|
||||
sb.WriteRune(r)
|
||||
}
|
||||
}
|
||||
sb.WriteByte('\'')
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
var numericLiteralRe = regexp.MustCompile(`^-?\d+(\.\d+)?$`)
|
||||
|
||||
func isNumericLiteral(s string) bool {
|
||||
return numericLiteralRe.MatchString(s)
|
||||
}
|
||||
Reference in New Issue
Block a user