Phase 2: unified query language spanning ClickHouse and Tantivy
Replaces the separate SQL-only /query and text-only /search endpoints with one pipe-syntax query language (plus raw SQL escape hatch) that compiles to a single IR and execution plan across both backends, so a query like `message:"connection refused" | stats count by host` runs as one request instead of two disjoint tools. - api/internal/querylang: lexer -> ast -> parser -> ir -> planner -> executor, each layer independently tested. - Execution generalizes Phase 1's proven Tantivy-prefilter pattern into a 4-way routing table (pure ClickHouse / text-only / text + aggregation / raw SQL passthrough). - Unified web query page and `sentryctl query`, both hitting the same POST /query endpoint. - Benchmarked against a real 1,022,000-row dataset (hack/benchmark-fixture); caught and fixed a real bug where the Tantivy prefilter cap (10,000) produced an IN-clause exceeding ClickHouse's default max_query_size -- lowered to 5,000, documented in docs/query-language-design.md and docs/phase-2-runbook.md. - docs/query-language-reference.md: customer-facing syntax reference.
This commit is contained in:
@@ -0,0 +1,91 @@
|
||||
// Package ast defines the parsed pipe-syntax tree. Internal to
|
||||
// querylang -- not exposed outside it. See
|
||||
// /docs/query-language-design.md for the grammar this mirrors.
|
||||
package ast
|
||||
|
||||
// Query is `base_search ("|" pipe_stage)*`.
|
||||
type Query struct {
|
||||
Base BoolExpr
|
||||
Pipes []PipeStage
|
||||
}
|
||||
|
||||
// PipeStage is one of WhereStage, StatsStage, SortStage, FieldsStage,
|
||||
// HeadStage, TailStage.
|
||||
type PipeStage interface{ isPipeStage() }
|
||||
|
||||
type WhereStage struct{ Expr BoolExpr }
|
||||
type StatsStage struct {
|
||||
Aggs []AggCall
|
||||
By []string
|
||||
}
|
||||
type SortStage struct{ Fields []SortField }
|
||||
type FieldsStage struct{ Fields []string }
|
||||
type HeadStage struct {
|
||||
N int
|
||||
HasN bool // false => default limit, decided by the planner
|
||||
}
|
||||
type TailStage struct {
|
||||
N int
|
||||
HasN bool
|
||||
}
|
||||
|
||||
func (WhereStage) isPipeStage() {}
|
||||
func (StatsStage) isPipeStage() {}
|
||||
func (SortStage) isPipeStage() {}
|
||||
func (FieldsStage) isPipeStage() {}
|
||||
func (HeadStage) isPipeStage() {}
|
||||
func (TailStage) isPipeStage() {}
|
||||
|
||||
// BoolExpr is a sequence of terms joined by "and"/"or". An empty Conjs
|
||||
// entry between two terms (i.e. no explicit keyword in the source) means
|
||||
// implicit AND -- SPL's convention for adjacent bare search terms, e.g.
|
||||
// `error timeout` means `error AND timeout`. Conjs has len(Terms)-1
|
||||
// elements once Terms has more than one.
|
||||
type BoolExpr struct {
|
||||
Terms []Term
|
||||
Conjs []string // "and" | "or", one per gap between consecutive Terms
|
||||
}
|
||||
|
||||
// Term is one of Comparison, TimeBound, FreeText.
|
||||
type Term interface{ isTerm() }
|
||||
|
||||
type Comparison struct {
|
||||
Field string
|
||||
Op string // "=", "!=", ">", ">=", "<", "<="
|
||||
Value string
|
||||
}
|
||||
|
||||
type TimeBound struct {
|
||||
Kind string // "earliest" | "latest"
|
||||
Expr TimeExpr
|
||||
}
|
||||
|
||||
type FreeText struct {
|
||||
Query string
|
||||
}
|
||||
|
||||
func (Comparison) isTerm() {}
|
||||
func (TimeBound) isTerm() {}
|
||||
func (FreeText) isTerm() {}
|
||||
|
||||
// TimeExpr is either an absolute RFC3339 timestamp or a relative offset
|
||||
// like -1h/-7d, resolved to an absolute time by the planner (relative to
|
||||
// compile time), not the parser -- the parser has no notion of "now".
|
||||
type TimeExpr struct {
|
||||
Absolute string
|
||||
IsRelative bool
|
||||
RelativeSign int // -1 or +1
|
||||
RelativeN int
|
||||
RelativeUnit string // "s" | "m" | "h" | "d" | "w"
|
||||
}
|
||||
|
||||
type AggCall struct {
|
||||
Func string // count, sum, avg, min, max
|
||||
Field string // empty for count()/count(*)
|
||||
Alias string // empty => planner assigns a default alias
|
||||
}
|
||||
|
||||
type SortField struct {
|
||||
Field string
|
||||
Desc bool
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package executor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
)
|
||||
|
||||
// ChRunner runs arbitrary (pre-validated) SELECT statements against
|
||||
// ClickHouse and shapes the result into JSON-friendly columns/rows,
|
||||
// discovering the result's column set at query time via reflection since
|
||||
// the query itself is arbitrary. Ported from Phase 0/1's
|
||||
// api/internal/queryapi.Executor, which this replaces (see task 4) --
|
||||
// same logic, moved here since it's the query-execution layer's
|
||||
// plumbing, not specific to the old placeholder /query handler.
|
||||
type ChRunner struct {
|
||||
conn driver.Conn
|
||||
}
|
||||
|
||||
func NewChRunner(conn driver.Conn) *ChRunner {
|
||||
return &ChRunner{conn: conn}
|
||||
}
|
||||
|
||||
func (r *ChRunner) RunSQL(ctx context.Context, sql string) (*Result, error) {
|
||||
rows, err := r.conn.Query(ctx, sql)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("executing query: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
columnTypes := rows.ColumnTypes()
|
||||
result := &Result{
|
||||
Columns: rows.Columns(),
|
||||
Rows: [][]any{},
|
||||
}
|
||||
|
||||
for rows.Next() {
|
||||
dest := make([]any, len(columnTypes))
|
||||
for i, ct := range columnTypes {
|
||||
dest[i] = reflect.New(ct.ScanType()).Interface()
|
||||
}
|
||||
if err := rows.Scan(dest...); err != nil {
|
||||
return nil, fmt.Errorf("scanning row: %w", err)
|
||||
}
|
||||
row := make([]any, len(dest))
|
||||
for i, d := range dest {
|
||||
row[i] = reflect.ValueOf(d).Elem().Interface()
|
||||
}
|
||||
result.Rows = append(result.Rows, row)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterating rows: %w", err)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
// Package executor runs a compiled ir.Plan and returns results in a
|
||||
// shape consistent regardless of which backend(s) were hit -- the point
|
||||
// of compiling to one IR in the first place. See
|
||||
// /docs/query-language-design.md's "Execution" section for the four
|
||||
// routing cases implemented here.
|
||||
package executor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/sentry/sentry/api/internal/querylang/ir"
|
||||
)
|
||||
|
||||
type Result struct {
|
||||
Columns []string
|
||||
Rows [][]any
|
||||
}
|
||||
|
||||
// SQLRunner executes a raw SQL statement against ClickHouse. *ChRunner
|
||||
// (chrunner.go) is the production implementation; tests use a fake --
|
||||
// same narrow-interface pattern used throughout /ingest and /api.
|
||||
type SQLRunner interface {
|
||||
RunSQL(ctx context.Context, sql string) (*Result, error)
|
||||
}
|
||||
|
||||
// SearchClient resolves a Tantivy query into matching record_ids.
|
||||
type SearchClient interface {
|
||||
Search(ctx context.Context, query string, limit uint32) ([]string, error)
|
||||
}
|
||||
|
||||
// textSearchLimit caps how many record_ids a Tantivy prefilter can feed
|
||||
// into a ClickHouse `IN (...)` clause. See /docs/query-language-design.md's
|
||||
// "Known scaling limitation" -- this is a real, disclosed limit on result
|
||||
// completeness for very broad text searches, not an oversight.
|
||||
//
|
||||
// 5000, not 10000: confirmed by actually running the Phase 2 benchmark
|
||||
// (see /docs/phase-2-runbook.md) that 10000 quoted UUIDs (~39 bytes each
|
||||
// including the comma) produces a ~390KB query string, which exceeds
|
||||
// ClickHouse's default max_query_size (262144 bytes / 256KiB) and fails
|
||||
// outright with a syntax error rather than degrading gracefully. 5000
|
||||
// UUIDs is ~195KB, safely under that default with headroom for the rest
|
||||
// of the query. This was a real failure caught by running the benchmark,
|
||||
// not a value chosen from first-principles estimation.
|
||||
const textSearchLimit = 5000
|
||||
|
||||
// Execute runs plan against the given backends. The four cases (per the
|
||||
// design doc): RawSQL passthrough; pure ClickHouse (no TextSearch); text
|
||||
// search alone (Tantivy prefilter -> ClickHouse row fetch); text search
|
||||
// plus aggregation (Tantivy prefilter -> ClickHouse aggregate). Cases 2-4
|
||||
// share the same buildSQL/buildWhereClause code (sql.go) -- the only
|
||||
// difference is whether a record_id filter is threaded in.
|
||||
func Execute(ctx context.Context, plan *ir.Plan, sqlRunner SQLRunner, search SearchClient) (*Result, error) {
|
||||
if plan.RawSQL != "" {
|
||||
return sqlRunner.RunSQL(ctx, plan.RawSQL)
|
||||
}
|
||||
|
||||
var recordIDFilter []string
|
||||
if len(plan.TextSearch) > 0 {
|
||||
ids, err := search.Search(ctx, plan.TextSearch[0].Query, textSearchLimit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("full-text search failed: %w", err)
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return &Result{Columns: []string{}, Rows: [][]any{}}, nil
|
||||
}
|
||||
recordIDFilter = ids
|
||||
}
|
||||
|
||||
sql := buildSQL(plan, recordIDFilter)
|
||||
return sqlRunner.RunSQL(ctx, sql)
|
||||
}
|
||||
@@ -0,0 +1,309 @@
|
||||
package executor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/sentry/sentry/api/internal/querylang/ir"
|
||||
)
|
||||
|
||||
func mustParseTime(t *testing.T, s string) time.Time {
|
||||
t.Helper()
|
||||
tm, err := time.Parse(time.RFC3339, s)
|
||||
if err != nil {
|
||||
t.Fatalf("parsing time %q: %v", s, err)
|
||||
}
|
||||
return tm
|
||||
}
|
||||
|
||||
type fakeSQLRunner struct {
|
||||
gotSQL string
|
||||
result *Result
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *fakeSQLRunner) RunSQL(_ context.Context, sql string) (*Result, error) {
|
||||
f.gotSQL = sql
|
||||
f.calls++
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
if f.result != nil {
|
||||
return f.result, nil
|
||||
}
|
||||
return &Result{Columns: []string{}, Rows: [][]any{}}, nil
|
||||
}
|
||||
|
||||
type fakeSearchClient struct {
|
||||
gotQuery string
|
||||
gotLimit uint32
|
||||
ids []string
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *fakeSearchClient) Search(_ context.Context, query string, limit uint32) ([]string, error) {
|
||||
f.gotQuery = query
|
||||
f.gotLimit = limit
|
||||
f.calls++
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
return f.ids, nil
|
||||
}
|
||||
|
||||
func TestExecuteRawSQLBypassesEverythingElse(t *testing.T) {
|
||||
sqlRunner := &fakeSQLRunner{}
|
||||
search := &fakeSearchClient{}
|
||||
plan := &ir.Plan{RawSQL: "SELECT 1"}
|
||||
|
||||
_, err := Execute(context.Background(), plan, sqlRunner, search)
|
||||
if err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
if sqlRunner.gotSQL != "SELECT 1" {
|
||||
t.Fatalf("gotSQL = %q, want %q", sqlRunner.gotSQL, "SELECT 1")
|
||||
}
|
||||
if search.calls != 0 {
|
||||
t.Fatalf("expected search not to be called for RawSQL, got %d calls", search.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutePureClickHousePathSkipsSearch(t *testing.T) {
|
||||
sqlRunner := &fakeSQLRunner{}
|
||||
search := &fakeSearchClient{}
|
||||
plan := &ir.Plan{Filters: []ir.FilterPredicate{{Field: "service", Op: "=", Value: "api"}}}
|
||||
|
||||
_, err := Execute(context.Background(), plan, sqlRunner, search)
|
||||
if err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
if search.calls != 0 {
|
||||
t.Fatalf("expected no search calls, got %d", search.calls)
|
||||
}
|
||||
if !strings.Contains(sqlRunner.gotSQL, "FROM logs") || !strings.Contains(sqlRunner.gotSQL, "`service` = 'api'") {
|
||||
t.Fatalf("unexpected SQL: %s", sqlRunner.gotSQL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteTextSearchPrefiltersThenQueriesClickHouse(t *testing.T) {
|
||||
sqlRunner := &fakeSQLRunner{}
|
||||
search := &fakeSearchClient{ids: []string{"id-1", "id-2"}}
|
||||
plan := &ir.Plan{TextSearch: []ir.TextPredicate{{Query: "connection refused"}}}
|
||||
|
||||
_, err := Execute(context.Background(), plan, sqlRunner, search)
|
||||
if err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
if search.gotQuery != "connection refused" {
|
||||
t.Fatalf("search query = %q", search.gotQuery)
|
||||
}
|
||||
if search.gotLimit != textSearchLimit {
|
||||
t.Fatalf("search limit = %d, want %d", search.gotLimit, textSearchLimit)
|
||||
}
|
||||
if !strings.Contains(sqlRunner.gotSQL, "record_id IN ('id-1','id-2')") {
|
||||
t.Fatalf("unexpected SQL: %s", sqlRunner.gotSQL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteTextSearchNoMatchesSkipsClickHouseEntirely(t *testing.T) {
|
||||
sqlRunner := &fakeSQLRunner{}
|
||||
search := &fakeSearchClient{ids: nil}
|
||||
plan := &ir.Plan{TextSearch: []ir.TextPredicate{{Query: "nothing matches"}}}
|
||||
|
||||
result, err := Execute(context.Background(), plan, sqlRunner, search)
|
||||
if err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
if sqlRunner.calls != 0 {
|
||||
t.Fatalf("expected ClickHouse not to be queried when search finds nothing, got %d calls", sqlRunner.calls)
|
||||
}
|
||||
if len(result.Columns) != 0 || len(result.Rows) != 0 {
|
||||
t.Fatalf("expected empty result, got %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteTextSearchWithAggregation(t *testing.T) {
|
||||
sqlRunner := &fakeSQLRunner{}
|
||||
search := &fakeSearchClient{ids: []string{"id-1"}}
|
||||
plan := &ir.Plan{
|
||||
TextSearch: []ir.TextPredicate{{Query: "connection refused"}},
|
||||
Aggregation: &ir.Aggregation{
|
||||
Funcs: []ir.AggFunc{{Func: "count", Alias: "count"}},
|
||||
GroupBy: []string{"host"},
|
||||
},
|
||||
}
|
||||
|
||||
_, err := Execute(context.Background(), plan, sqlRunner, search)
|
||||
if err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(sqlRunner.gotSQL, "record_id IN ('id-1')") {
|
||||
t.Fatalf("expected the text-search prefilter in the WHERE clause: %s", sqlRunner.gotSQL)
|
||||
}
|
||||
if !strings.Contains(sqlRunner.gotSQL, "GROUP BY `host`") {
|
||||
t.Fatalf("expected GROUP BY: %s", sqlRunner.gotSQL)
|
||||
}
|
||||
if !strings.Contains(sqlRunner.gotSQL, "count() AS `count`") {
|
||||
t.Fatalf("expected count() AS `count`: %s", sqlRunner.gotSQL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteSearchErrorPropagates(t *testing.T) {
|
||||
sqlRunner := &fakeSQLRunner{}
|
||||
search := &fakeSearchClient{err: errors.New("search unavailable")}
|
||||
plan := &ir.Plan{TextSearch: []ir.TextPredicate{{Query: "x"}}}
|
||||
|
||||
_, err := Execute(context.Background(), plan, sqlRunner, search)
|
||||
if err == nil {
|
||||
t.Fatal("expected the search error to propagate")
|
||||
}
|
||||
if sqlRunner.calls != 0 {
|
||||
t.Fatalf("expected ClickHouse not to be queried after a search error, got %d calls", sqlRunner.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSQLNumericCastOnAttributesField(t *testing.T) {
|
||||
plan := &ir.Plan{Filters: []ir.FilterPredicate{{Field: "status", Op: ">=", Value: "500"}}}
|
||||
sql := buildSQL(plan, nil)
|
||||
want := "toFloat64OrZero(attributes['status']) >= 500"
|
||||
if !strings.Contains(sql, want) {
|
||||
t.Fatalf("SQL = %q, want it to contain %q", sql, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSQLStringComparisonOnAttributesField(t *testing.T) {
|
||||
plan := &ir.Plan{Filters: []ir.FilterPredicate{{Field: "status", Op: "=", Value: "unknown"}}}
|
||||
sql := buildSQL(plan, nil)
|
||||
want := "attributes['status'] = 'unknown'"
|
||||
if !strings.Contains(sql, want) {
|
||||
t.Fatalf("SQL = %q, want it to contain %q", sql, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSQLTopLevelFieldNeverCast(t *testing.T) {
|
||||
plan := &ir.Plan{Filters: []ir.FilterPredicate{{Field: "service", Op: "=", Value: "123"}}}
|
||||
sql := buildSQL(plan, nil)
|
||||
if strings.Contains(sql, "toFloat64OrZero") {
|
||||
t.Fatalf("top-level field should never be numeric-cast: %s", sql)
|
||||
}
|
||||
if !strings.Contains(sql, "`service` = '123'") {
|
||||
t.Fatalf("unexpected SQL: %s", sql)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSQLEscapesInjectionAttemptInValue(t *testing.T) {
|
||||
plan := &ir.Plan{Filters: []ir.FilterPredicate{{Field: "service", Op: "=", Value: "x'; DROP TABLE logs; --"}}}
|
||||
sql := buildSQL(plan, nil)
|
||||
// The whole attacker-controlled value must land inside exactly one
|
||||
// quoted literal, with its embedded quote backslash-escaped so it
|
||||
// can't terminate the literal early -- checking for the escaped
|
||||
// form directly, not just the absence of the raw substring (which
|
||||
// is a weaker check: "\\'; DROP TABLE" still *contains* "'; DROP
|
||||
// TABLE" as a substring, so that alone doesn't prove escaping
|
||||
// happened).
|
||||
want := `'x\'; DROP TABLE logs; --'`
|
||||
if !strings.Contains(sql, want) {
|
||||
t.Fatalf("expected the literal %q in SQL, got: %s", want, sql)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSQLDefaultLimitAppliedWhenNoneGiven(t *testing.T) {
|
||||
plan := &ir.Plan{Filters: []ir.FilterPredicate{{Field: "service", Op: "=", Value: "api"}}}
|
||||
sql := buildSQL(plan, nil)
|
||||
if !strings.Contains(sql, "LIMIT 100") {
|
||||
t.Fatalf("expected the default row limit, got: %s", sql)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSQLExplicitLimitOverridesDefault(t *testing.T) {
|
||||
plan := &ir.Plan{
|
||||
Filters: []ir.FilterPredicate{{Field: "service", Op: "=", Value: "api"}},
|
||||
Limit: &ir.Limit{N: 5},
|
||||
}
|
||||
sql := buildSQL(plan, nil)
|
||||
if !strings.Contains(sql, "LIMIT 5") || strings.Contains(sql, "LIMIT 100") {
|
||||
t.Fatalf("expected LIMIT 5, got: %s", sql)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSQLTailWithoutSortOrdersAscending(t *testing.T) {
|
||||
plan := &ir.Plan{Limit: &ir.Limit{N: 10, Tail: true}}
|
||||
sql := buildSQL(plan, nil)
|
||||
if !strings.Contains(sql, "ORDER BY `timestamp` ASC") {
|
||||
t.Fatalf("expected ascending order for tail, got: %s", sql)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSQLNoSortDefaultsNewestFirst(t *testing.T) {
|
||||
plan := &ir.Plan{}
|
||||
sql := buildSQL(plan, nil)
|
||||
if !strings.Contains(sql, "ORDER BY `timestamp` DESC") {
|
||||
t.Fatalf("expected newest-first default, got: %s", sql)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSQLSortByAggregateAlias(t *testing.T) {
|
||||
plan := &ir.Plan{
|
||||
Aggregation: &ir.Aggregation{
|
||||
Funcs: []ir.AggFunc{{Func: "count", Alias: "count"}},
|
||||
GroupBy: []string{"host"},
|
||||
},
|
||||
Sort: []ir.SortField{{Field: "count", Desc: true}},
|
||||
}
|
||||
sql := buildSQL(plan, nil)
|
||||
if !strings.Contains(sql, "ORDER BY `count` DESC") {
|
||||
t.Fatalf("expected ORDER BY on the aggregate alias, got: %s", sql)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSQLSortByGroupByField(t *testing.T) {
|
||||
plan := &ir.Plan{
|
||||
Aggregation: &ir.Aggregation{
|
||||
Funcs: []ir.AggFunc{{Func: "count", Alias: "count"}},
|
||||
GroupBy: []string{"host"},
|
||||
},
|
||||
Sort: []ir.SortField{{Field: "host", Desc: false}},
|
||||
}
|
||||
sql := buildSQL(plan, nil)
|
||||
if !strings.Contains(sql, "ORDER BY `host` ASC") {
|
||||
t.Fatalf("expected ORDER BY on the group-by column, got: %s", sql)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSQLAggregationOnAttributesFieldAlwaysCasts(t *testing.T) {
|
||||
plan := &ir.Plan{
|
||||
Aggregation: &ir.Aggregation{
|
||||
Funcs: []ir.AggFunc{{Func: "avg", Field: "latency_ms", Alias: "avg_latency"}},
|
||||
},
|
||||
}
|
||||
sql := buildSQL(plan, nil)
|
||||
if !strings.Contains(sql, "AVG(toFloat64OrZero(attributes['latency_ms'])) AS `avg_latency`") {
|
||||
t.Fatalf("unexpected SQL: %s", sql)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSQLProjectionFields(t *testing.T) {
|
||||
plan := &ir.Plan{Fields: []string{"host", "message"}}
|
||||
sql := buildSQL(plan, nil)
|
||||
if !strings.Contains(sql, "SELECT `host` AS `host`, `message` AS `message` FROM logs") {
|
||||
t.Fatalf("unexpected SQL: %s", sql)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSQLTimeRange(t *testing.T) {
|
||||
from := mustParseTime(t, "2026-08-14T00:00:00Z")
|
||||
to := mustParseTime(t, "2026-08-14T01:00:00Z")
|
||||
plan := &ir.Plan{TimeRange: &ir.TimeRange{From: from, To: to}}
|
||||
sql := buildSQL(plan, nil)
|
||||
if !strings.Contains(sql, "`timestamp` >= '2026-08-14T00:00:00Z'") {
|
||||
t.Fatalf("missing From bound: %s", sql)
|
||||
}
|
||||
if !strings.Contains(sql, "`timestamp` <= '2026-08-14T01:00:00Z'") {
|
||||
t.Fatalf("missing To bound: %s", sql)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
package executor
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/sentry/sentry/api/internal/querylang/ir"
|
||||
)
|
||||
|
||||
// defaultRowLimit is the safety net when a raw-row query has neither an
|
||||
// explicit head/tail nor an aggregation -- without it, a bare `service=api`
|
||||
// with no other pipe stages would return every matching row unbounded.
|
||||
// Independent of planner's own defaultLimit (same value, different
|
||||
// concern: that one fills in `head`/`tail` with no N given; this one
|
||||
// guards queries that never mention head/tail at all).
|
||||
const defaultRowLimit = 100
|
||||
|
||||
// logs' real columns, per /storage. Anything else maps to
|
||||
// attributes['field'] -- see /docs/query-language-design.md's "Field
|
||||
// mapping" section.
|
||||
var topLevelFields = map[string]bool{
|
||||
"timestamp": true,
|
||||
"host": true,
|
||||
"service": true,
|
||||
"severity": true,
|
||||
"message": true,
|
||||
"record_id": true,
|
||||
}
|
||||
|
||||
func buildSQL(plan *ir.Plan, recordIDFilter []string) string {
|
||||
var sb strings.Builder
|
||||
|
||||
sb.WriteString("SELECT ")
|
||||
sb.WriteString(selectClause(plan))
|
||||
sb.WriteString(" FROM logs")
|
||||
|
||||
if where := buildWhereClause(plan, recordIDFilter); where != "" {
|
||||
sb.WriteString(" WHERE ")
|
||||
sb.WriteString(where)
|
||||
}
|
||||
|
||||
if plan.Aggregation != nil && len(plan.Aggregation.GroupBy) > 0 {
|
||||
sb.WriteString(" GROUP BY ")
|
||||
cols := make([]string, len(plan.Aggregation.GroupBy))
|
||||
for i, g := range plan.Aggregation.GroupBy {
|
||||
cols[i] = columnExpr(g)
|
||||
}
|
||||
sb.WriteString(strings.Join(cols, ", "))
|
||||
}
|
||||
|
||||
writeOrderBy(&sb, plan)
|
||||
|
||||
if plan.Limit != nil {
|
||||
fmt.Fprintf(&sb, " LIMIT %d", plan.Limit.N)
|
||||
} else if plan.Aggregation == nil {
|
||||
fmt.Fprintf(&sb, " LIMIT %d", defaultRowLimit)
|
||||
}
|
||||
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func writeOrderBy(sb *strings.Builder, plan *ir.Plan) {
|
||||
switch {
|
||||
case len(plan.Sort) > 0:
|
||||
sb.WriteString(" ORDER BY ")
|
||||
parts := make([]string, len(plan.Sort))
|
||||
for i, s := range plan.Sort {
|
||||
dir := "ASC"
|
||||
if s.Desc {
|
||||
dir = "DESC"
|
||||
}
|
||||
parts[i] = sortColumnExpr(plan, s.Field) + " " + dir
|
||||
}
|
||||
sb.WriteString(strings.Join(parts, ", "))
|
||||
case plan.Limit != nil && plan.Limit.Tail:
|
||||
// `tail N` with no explicit sort: order ascending so LIMIT N
|
||||
// takes the chronologically *last* N rows. Callers wanting
|
||||
// strict newest-first display order re-sort client-side --
|
||||
// documented in the query language reference.
|
||||
sb.WriteString(" ORDER BY `timestamp` ASC")
|
||||
case plan.Aggregation == nil:
|
||||
// Raw-row queries with no explicit sort default to newest-first,
|
||||
// matching the Phase 0/1 UI default.
|
||||
sb.WriteString(" ORDER BY `timestamp` DESC")
|
||||
}
|
||||
}
|
||||
|
||||
// sortColumnExpr resolves a sort field against an aggregation's own
|
||||
// output columns (alias or group-by field) before falling back to the
|
||||
// normal top-level/attributes mapping -- `sort -count` after `stats
|
||||
// count` refers to the aggregate's alias, not a raw column.
|
||||
func sortColumnExpr(plan *ir.Plan, field string) string {
|
||||
if plan.Aggregation != nil {
|
||||
for _, f := range plan.Aggregation.Funcs {
|
||||
if f.Alias == field {
|
||||
return quoteIdent(field)
|
||||
}
|
||||
}
|
||||
for _, g := range plan.Aggregation.GroupBy {
|
||||
if g == field {
|
||||
return columnExpr(field)
|
||||
}
|
||||
}
|
||||
}
|
||||
return columnExpr(field)
|
||||
}
|
||||
|
||||
func selectClause(plan *ir.Plan) string {
|
||||
if plan.Aggregation != nil {
|
||||
parts := make([]string, 0, len(plan.Aggregation.GroupBy)+len(plan.Aggregation.Funcs))
|
||||
for _, g := range plan.Aggregation.GroupBy {
|
||||
parts = append(parts, columnExpr(g)+" AS "+quoteIdent(g))
|
||||
}
|
||||
for _, f := range plan.Aggregation.Funcs {
|
||||
parts = append(parts, aggExpr(f)+" AS "+quoteIdent(f.Alias))
|
||||
}
|
||||
return strings.Join(parts, ", ")
|
||||
}
|
||||
if len(plan.Fields) > 0 {
|
||||
parts := make([]string, len(plan.Fields))
|
||||
for i, f := range plan.Fields {
|
||||
parts[i] = columnExpr(f) + " AS " + quoteIdent(f)
|
||||
}
|
||||
return strings.Join(parts, ", ")
|
||||
}
|
||||
return "*"
|
||||
}
|
||||
|
||||
// aggExpr always numeric-casts non-top-level (attributes-map) fields for
|
||||
// sum/avg/min/max, unlike comparison predicates where casting is
|
||||
// conditional on whether the compared value looks numeric -- an
|
||||
// aggregate function is inherently a numeric (or, for min/max,
|
||||
// order-comparable) operation, so there's no "maybe string" case the way
|
||||
// there is for `field=value`. Known Phase 2 limitation: min/max on a
|
||||
// non-top-level field always compares numerically, not lexicographically
|
||||
// -- string min/max on attributes isn't supported this phase.
|
||||
func aggExpr(f ir.AggFunc) string {
|
||||
if f.Func == "count" {
|
||||
return "count()"
|
||||
}
|
||||
col := columnExpr(f.Field)
|
||||
if !topLevelFields[f.Field] {
|
||||
col = "toFloat64OrZero(" + col + ")"
|
||||
}
|
||||
return strings.ToUpper(f.Func) + "(" + col + ")"
|
||||
}
|
||||
|
||||
func buildWhereClause(plan *ir.Plan, recordIDFilter []string) string {
|
||||
var conds []string
|
||||
|
||||
if len(recordIDFilter) > 0 {
|
||||
quoted := make([]string, len(recordIDFilter))
|
||||
for i, id := range recordIDFilter {
|
||||
quoted[i] = quoteLiteral(id)
|
||||
}
|
||||
conds = append(conds, "record_id IN ("+strings.Join(quoted, ",")+")")
|
||||
}
|
||||
|
||||
for _, f := range plan.Filters {
|
||||
conds = append(conds, buildComparisonSQL(f))
|
||||
}
|
||||
|
||||
if plan.TimeRange != nil {
|
||||
if !plan.TimeRange.From.IsZero() {
|
||||
conds = append(conds, "`timestamp` >= "+quoteLiteral(plan.TimeRange.From.UTC().Format(time.RFC3339Nano)))
|
||||
}
|
||||
if !plan.TimeRange.To.IsZero() {
|
||||
conds = append(conds, "`timestamp` <= "+quoteLiteral(plan.TimeRange.To.UTC().Format(time.RFC3339Nano)))
|
||||
}
|
||||
}
|
||||
|
||||
return strings.Join(conds, " AND ")
|
||||
}
|
||||
|
||||
// buildComparisonSQL numeric-casts a non-top-level field only when the
|
||||
// compared value itself looks numeric -- `status>=500` casts (numeric
|
||||
// comparison intent), `status="unknown"` doesn't (string comparison
|
||||
// intent). Top-level fields are never cast; ClickHouse compares them
|
||||
// against a string literal natively (DateTime64 columns parse an
|
||||
// RFC3339-shaped literal, LowCardinality(String)/String compare as-is).
|
||||
func buildComparisonSQL(f ir.FilterPredicate) string {
|
||||
if !topLevelFields[f.Field] && isNumericLiteral(f.Value) {
|
||||
return "toFloat64OrZero(" + columnExpr(f.Field) + ") " + f.Op + " " + f.Value
|
||||
}
|
||||
return columnExpr(f.Field) + " " + f.Op + " " + quoteLiteral(f.Value)
|
||||
}
|
||||
|
||||
func columnExpr(field string) string {
|
||||
if topLevelFields[field] {
|
||||
return quoteIdent(field)
|
||||
}
|
||||
return "attributes[" + quoteLiteral(field) + "]"
|
||||
}
|
||||
|
||||
func quoteIdent(name string) string {
|
||||
return "`" + strings.ReplaceAll(name, "`", "``") + "`"
|
||||
}
|
||||
|
||||
// quoteLiteral is the actual injection defense for every user-controlled
|
||||
// string embedded in generated SQL (filter values, attribute keys, time
|
||||
// bounds, record_ids). Field/keyword tokens from the lexer are already
|
||||
// constrained to [a-zA-Z0-9_.] by construction (see lexer.isIdentPart)
|
||||
// and can't carry SQL metacharacters at all, but quoted-string *values*
|
||||
// can contain anything, so this can't be skipped for them.
|
||||
func quoteLiteral(s string) string {
|
||||
var sb strings.Builder
|
||||
sb.WriteByte('\'')
|
||||
for _, r := range s {
|
||||
switch r {
|
||||
case '\\':
|
||||
sb.WriteString(`\\`)
|
||||
case '\'':
|
||||
sb.WriteString(`\'`)
|
||||
default:
|
||||
sb.WriteRune(r)
|
||||
}
|
||||
}
|
||||
sb.WriteByte('\'')
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
var numericLiteralRe = regexp.MustCompile(`^-?\d+(\.\d+)?$`)
|
||||
|
||||
func isNumericLiteral(s string) bool {
|
||||
return numericLiteralRe.MatchString(s)
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
// Package ir defines Plan, the intermediate representation both the
|
||||
// pipe-syntax parser+planner and the raw-SQL passthrough compile down
|
||||
// to. This is the boundary task 3 asked for: "pipe syntax X compiles to
|
||||
// IR Y" is testable in planner without any backend; "IR Y executes
|
||||
// correctly" is testable in executor against fakes, independent of the
|
||||
// planner. See /docs/query-language-design.md.
|
||||
package ir
|
||||
|
||||
import "time"
|
||||
|
||||
type Plan struct {
|
||||
// RawSQL, when non-empty, means the entire plan is this opaque
|
||||
// ClickHouse SQL string, executed as-is -- every other field below
|
||||
// is unused. This is the SQL escape hatch's IR representation: a
|
||||
// trivial identity compilation that still flows through the same
|
||||
// Plan type and the same executor code path as a parsed pipe query.
|
||||
RawSQL string
|
||||
|
||||
// TextSearch predicates route to Tantivy as a prefilter. Empty means
|
||||
// no Tantivy involvement at all -- pure ClickHouse.
|
||||
TextSearch []TextPredicate
|
||||
|
||||
// Filters are always evaluated in ClickHouse, either directly as
|
||||
// WHERE clauses (no TextSearch present) or as an additional filter
|
||||
// alongside a Tantivy-sourced record_id IN (...) clause.
|
||||
Filters []FilterPredicate
|
||||
|
||||
TimeRange *TimeRange
|
||||
|
||||
// Aggregation is nil for a raw-rows query (no GROUP BY).
|
||||
Aggregation *Aggregation
|
||||
|
||||
Sort []SortField
|
||||
|
||||
// Fields is the projection; empty means all columns.
|
||||
Fields []string
|
||||
|
||||
Limit *Limit
|
||||
}
|
||||
|
||||
type TextPredicate struct {
|
||||
// Query is passed to Tantivy's query parser as-is -- phrase and
|
||||
// wildcard syntax already supported there (see /search).
|
||||
Query string
|
||||
}
|
||||
|
||||
type FilterPredicate struct {
|
||||
Field string
|
||||
Op string // "=", "!=", ">", ">=", "<", "<="
|
||||
Value string
|
||||
}
|
||||
|
||||
type Aggregation struct {
|
||||
Funcs []AggFunc
|
||||
GroupBy []string
|
||||
}
|
||||
|
||||
type AggFunc struct {
|
||||
Func string // count, sum, avg, min, max
|
||||
Field string // empty for count
|
||||
Alias string // always set by the planner (defaulted if not given explicitly)
|
||||
}
|
||||
|
||||
type SortField struct {
|
||||
Field string
|
||||
Desc bool
|
||||
}
|
||||
|
||||
type Limit struct {
|
||||
N int
|
||||
Tail bool // true = last N (by time), false = first N
|
||||
}
|
||||
|
||||
type TimeRange struct {
|
||||
// Absolute bounds -- any relative expression (-1h etc.) is resolved
|
||||
// by the planner at compile time, since only it knows "now". A zero
|
||||
// time.Time means that bound is unset.
|
||||
From time.Time
|
||||
To time.Time
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
// Package lexer tokenizes the pipe-syntax query language. Deliberately
|
||||
// simple: keywords (where/stats/sort/and/etc.) aren't distinct token
|
||||
// kinds -- they're just Ident tokens whose value the parser checks
|
||||
// against a keyword set, so the lexer stays context-free and the parser
|
||||
// owns all the grammar decisions. See /docs/query-language-design.md.
|
||||
package lexer
|
||||
|
||||
import "fmt"
|
||||
|
||||
type Kind int
|
||||
|
||||
const (
|
||||
EOF Kind = iota
|
||||
Illegal
|
||||
Ident // bare words: field names, keywords, unquoted values/free-text terms
|
||||
String // quoted string: "..."
|
||||
Number // 123, 1.5
|
||||
Pipe // |
|
||||
Eq // =
|
||||
Neq // !=
|
||||
Gt // >
|
||||
Gte // >=
|
||||
Lt // <
|
||||
Lte // <=
|
||||
Colon // :
|
||||
Comma // ,
|
||||
LParen // (
|
||||
RParen // )
|
||||
Minus // -
|
||||
Plus // +
|
||||
Star // * (only meaningful inside count(*), same as SQL)
|
||||
)
|
||||
|
||||
type Token struct {
|
||||
Kind Kind
|
||||
Value string
|
||||
Pos int // byte offset into the original input, for error messages
|
||||
}
|
||||
|
||||
func (t Token) String() string {
|
||||
return fmt.Sprintf("%s(%q)@%d", t.Kind, t.Value, t.Pos)
|
||||
}
|
||||
|
||||
func (k Kind) String() string {
|
||||
switch k {
|
||||
case EOF:
|
||||
return "EOF"
|
||||
case Illegal:
|
||||
return "ILLEGAL"
|
||||
case Ident:
|
||||
return "IDENT"
|
||||
case String:
|
||||
return "STRING"
|
||||
case Number:
|
||||
return "NUMBER"
|
||||
case Pipe:
|
||||
return "PIPE"
|
||||
case Eq:
|
||||
return "EQ"
|
||||
case Neq:
|
||||
return "NEQ"
|
||||
case Gt:
|
||||
return "GT"
|
||||
case Gte:
|
||||
return "GTE"
|
||||
case Lt:
|
||||
return "LT"
|
||||
case Lte:
|
||||
return "LTE"
|
||||
case Colon:
|
||||
return "COLON"
|
||||
case Comma:
|
||||
return "COMMA"
|
||||
case LParen:
|
||||
return "LPAREN"
|
||||
case RParen:
|
||||
return "RPAREN"
|
||||
case Minus:
|
||||
return "MINUS"
|
||||
case Plus:
|
||||
return "PLUS"
|
||||
case Star:
|
||||
return "STAR"
|
||||
default:
|
||||
return "UNKNOWN"
|
||||
}
|
||||
}
|
||||
|
||||
type Lexer struct {
|
||||
input []rune
|
||||
pos int
|
||||
}
|
||||
|
||||
func New(input string) *Lexer {
|
||||
return &Lexer{input: []rune(input)}
|
||||
}
|
||||
|
||||
func (l *Lexer) Next() Token {
|
||||
l.skipWhitespace()
|
||||
if l.pos >= len(l.input) {
|
||||
return Token{Kind: EOF, Pos: l.pos}
|
||||
}
|
||||
|
||||
start := l.pos
|
||||
c := l.input[l.pos]
|
||||
|
||||
switch {
|
||||
case c == '|':
|
||||
l.pos++
|
||||
return Token{Kind: Pipe, Value: "|", Pos: start}
|
||||
case c == '=':
|
||||
l.pos++
|
||||
return Token{Kind: Eq, Value: "=", Pos: start}
|
||||
case c == '!' && l.peek(1) == '=':
|
||||
l.pos += 2
|
||||
return Token{Kind: Neq, Value: "!=", Pos: start}
|
||||
case c == '>' && l.peek(1) == '=':
|
||||
l.pos += 2
|
||||
return Token{Kind: Gte, Value: ">=", Pos: start}
|
||||
case c == '>':
|
||||
l.pos++
|
||||
return Token{Kind: Gt, Value: ">", Pos: start}
|
||||
case c == '<' && l.peek(1) == '=':
|
||||
l.pos += 2
|
||||
return Token{Kind: Lte, Value: "<=", Pos: start}
|
||||
case c == '<':
|
||||
l.pos++
|
||||
return Token{Kind: Lt, Value: "<", Pos: start}
|
||||
case c == ':':
|
||||
l.pos++
|
||||
return Token{Kind: Colon, Value: ":", Pos: start}
|
||||
case c == ',':
|
||||
l.pos++
|
||||
return Token{Kind: Comma, Value: ",", Pos: start}
|
||||
case c == '(':
|
||||
l.pos++
|
||||
return Token{Kind: LParen, Value: "(", Pos: start}
|
||||
case c == ')':
|
||||
l.pos++
|
||||
return Token{Kind: RParen, Value: ")", Pos: start}
|
||||
case c == '-':
|
||||
l.pos++
|
||||
return Token{Kind: Minus, Value: "-", Pos: start}
|
||||
case c == '+':
|
||||
l.pos++
|
||||
return Token{Kind: Plus, Value: "+", Pos: start}
|
||||
case c == '*':
|
||||
l.pos++
|
||||
return Token{Kind: Star, Value: "*", Pos: start}
|
||||
case c == '"':
|
||||
return l.lexString()
|
||||
case isDigit(c):
|
||||
return l.lexNumber()
|
||||
case isIdentStart(c):
|
||||
return l.lexIdent()
|
||||
default:
|
||||
l.pos++
|
||||
return Token{Kind: Illegal, Value: string(c), Pos: start}
|
||||
}
|
||||
}
|
||||
|
||||
func (l *Lexer) peek(offset int) rune {
|
||||
p := l.pos + offset
|
||||
if p >= len(l.input) {
|
||||
return 0
|
||||
}
|
||||
return l.input[p]
|
||||
}
|
||||
|
||||
func (l *Lexer) skipWhitespace() {
|
||||
for l.pos < len(l.input) {
|
||||
switch l.input[l.pos] {
|
||||
case ' ', '\t', '\n', '\r':
|
||||
l.pos++
|
||||
default:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (l *Lexer) lexString() Token {
|
||||
start := l.pos
|
||||
l.pos++ // consume opening quote
|
||||
var sb []rune
|
||||
for l.pos < len(l.input) {
|
||||
c := l.input[l.pos]
|
||||
if c == '"' {
|
||||
l.pos++
|
||||
return Token{Kind: String, Value: string(sb), Pos: start}
|
||||
}
|
||||
if c == '\\' && l.pos+1 < len(l.input) {
|
||||
l.pos++
|
||||
sb = append(sb, l.input[l.pos])
|
||||
l.pos++
|
||||
continue
|
||||
}
|
||||
sb = append(sb, c)
|
||||
l.pos++
|
||||
}
|
||||
// unterminated string -- return what we have as Illegal so the
|
||||
// parser can produce a clear "unterminated string" error rather than
|
||||
// the lexer silently accepting it.
|
||||
return Token{Kind: Illegal, Value: string(sb), Pos: start}
|
||||
}
|
||||
|
||||
func (l *Lexer) lexNumber() Token {
|
||||
start := l.pos
|
||||
for l.pos < len(l.input) && isDigit(l.input[l.pos]) {
|
||||
l.pos++
|
||||
}
|
||||
if l.pos < len(l.input) && l.input[l.pos] == '.' && l.pos+1 < len(l.input) && isDigit(l.input[l.pos+1]) {
|
||||
l.pos++
|
||||
for l.pos < len(l.input) && isDigit(l.input[l.pos]) {
|
||||
l.pos++
|
||||
}
|
||||
}
|
||||
return Token{Kind: Number, Value: string(l.input[start:l.pos]), Pos: start}
|
||||
}
|
||||
|
||||
func (l *Lexer) lexIdent() Token {
|
||||
start := l.pos
|
||||
for l.pos < len(l.input) && isIdentPart(l.input[l.pos]) {
|
||||
l.pos++
|
||||
}
|
||||
return Token{Kind: Ident, Value: string(l.input[start:l.pos]), Pos: start}
|
||||
}
|
||||
|
||||
func isDigit(c rune) bool { return c >= '0' && c <= '9' }
|
||||
|
||||
func isIdentStart(c rune) bool {
|
||||
return c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
|
||||
}
|
||||
|
||||
// Identifiers allow dots (e.g. winevt.event_id, a real attribute key
|
||||
// shape from Phase 1) and digits after the first character.
|
||||
func isIdentPart(c rune) bool {
|
||||
return isIdentStart(c) || isDigit(c) || c == '.' || c == '_'
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package lexer
|
||||
|
||||
import "testing"
|
||||
|
||||
func collectKinds(input string) []Kind {
|
||||
l := New(input)
|
||||
var kinds []Kind
|
||||
for {
|
||||
tok := l.Next()
|
||||
kinds = append(kinds, tok.Kind)
|
||||
if tok.Kind == EOF {
|
||||
return kinds
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLexSimpleFilter(t *testing.T) {
|
||||
got := collectKinds(`service=api`)
|
||||
want := []Kind{Ident, Eq, Ident, EOF}
|
||||
assertKinds(t, got, want)
|
||||
}
|
||||
|
||||
func TestLexPipeline(t *testing.T) {
|
||||
got := collectKinds(`service=api | where status>=500 | stats count(*) by host`)
|
||||
want := []Kind{
|
||||
Ident, Eq, Ident, Pipe,
|
||||
Ident, Ident, Gte, Number, Pipe,
|
||||
Ident, Ident, LParen, Star, RParen, Ident, Ident,
|
||||
EOF,
|
||||
}
|
||||
assertKinds(t, got, want)
|
||||
}
|
||||
|
||||
func TestLexOperators(t *testing.T) {
|
||||
got := collectKinds(`= != > >= < <=`)
|
||||
want := []Kind{Eq, Neq, Gt, Gte, Lt, Lte, EOF}
|
||||
assertKinds(t, got, want)
|
||||
}
|
||||
|
||||
func TestLexQuotedString(t *testing.T) {
|
||||
l := New(`"connection refused"`)
|
||||
tok := l.Next()
|
||||
if tok.Kind != String {
|
||||
t.Fatalf("Kind = %v, want String", tok.Kind)
|
||||
}
|
||||
if tok.Value != "connection refused" {
|
||||
t.Fatalf("Value = %q, want %q", tok.Value, "connection refused")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLexQuotedStringWithEscapes(t *testing.T) {
|
||||
l := New(`"has \"quotes\" and \\backslash"`)
|
||||
tok := l.Next()
|
||||
if tok.Kind != String {
|
||||
t.Fatalf("Kind = %v, want String", tok.Kind)
|
||||
}
|
||||
want := `has "quotes" and \backslash`
|
||||
if tok.Value != want {
|
||||
t.Fatalf("Value = %q, want %q", tok.Value, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLexUnterminatedStringIsIllegal(t *testing.T) {
|
||||
l := New(`"unterminated`)
|
||||
tok := l.Next()
|
||||
if tok.Kind != Illegal {
|
||||
t.Fatalf("Kind = %v, want Illegal", tok.Kind)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLexFieldWithDots(t *testing.T) {
|
||||
l := New(`winevt.event_id=4625`)
|
||||
tok := l.Next()
|
||||
if tok.Kind != Ident || tok.Value != "winevt.event_id" {
|
||||
t.Fatalf("got %v, want Ident(winevt.event_id)", tok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLexNumber(t *testing.T) {
|
||||
cases := []string{"123", "1.5", "0"}
|
||||
for _, c := range cases {
|
||||
l := New(c)
|
||||
tok := l.Next()
|
||||
if tok.Kind != Number || tok.Value != c {
|
||||
t.Errorf("lexing %q: got %v, want Number(%s)", c, tok, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLexNegativeTimeExpr(t *testing.T) {
|
||||
// "-1h" lexes as MINUS, NUMBER, IDENT -- the parser composes these,
|
||||
// not the lexer (see package doc comment).
|
||||
got := collectKinds(`-1h`)
|
||||
want := []Kind{Minus, Number, Ident, EOF}
|
||||
assertKinds(t, got, want)
|
||||
}
|
||||
|
||||
func TestLexWhitespaceIsSkipped(t *testing.T) {
|
||||
got := collectKinds(" service = api ")
|
||||
want := []Kind{Ident, Eq, Ident, EOF}
|
||||
assertKinds(t, got, want)
|
||||
}
|
||||
|
||||
func TestLexEmptyInput(t *testing.T) {
|
||||
got := collectKinds("")
|
||||
want := []Kind{EOF}
|
||||
assertKinds(t, got, want)
|
||||
}
|
||||
|
||||
func TestLexIllegalCharacter(t *testing.T) {
|
||||
l := New(`$`)
|
||||
tok := l.Next()
|
||||
if tok.Kind != Illegal {
|
||||
t.Fatalf("Kind = %v, want Illegal", tok.Kind)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTokenPositionsAreByteOffsets(t *testing.T) {
|
||||
l := New(`service=api`)
|
||||
first := l.Next()
|
||||
second := l.Next()
|
||||
if first.Pos != 0 {
|
||||
t.Errorf("first.Pos = %d, want 0", first.Pos)
|
||||
}
|
||||
if second.Pos != 7 {
|
||||
t.Errorf("second.Pos = %d, want 7", second.Pos)
|
||||
}
|
||||
}
|
||||
|
||||
func assertKinds(t *testing.T, got, want []Kind) {
|
||||
t.Helper()
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("got %d tokens %v, want %d tokens %v", len(got), got, len(want), want)
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("token %d: got %v, want %v (full: got=%v want=%v)", i, got[i], want[i], got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,452 @@
|
||||
// Package parser is a hand-written recursive-descent parser for the
|
||||
// pipe-syntax query language, per /docs/query-language-design.md's
|
||||
// choice of parser approach (no combinator/generator library -- this
|
||||
// grammar is small and stable, and hand-written gives full control over
|
||||
// error messages, which matter for a user-facing query language).
|
||||
package parser
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"github.com/sentry/sentry/api/internal/querylang/ast"
|
||||
"github.com/sentry/sentry/api/internal/querylang/lexer"
|
||||
)
|
||||
|
||||
// Parse parses a pipe-syntax query. Callers are responsible for routing
|
||||
// SQL (queries starting with "SELECT") elsewhere before calling this --
|
||||
// see planner.Plan and /docs/query-language-design.md's "SQL escape
|
||||
// hatch" section for why this parser never sees SQL at all.
|
||||
func Parse(input string) (*ast.Query, error) {
|
||||
p := newParser(input)
|
||||
return p.parseQuery()
|
||||
}
|
||||
|
||||
type parser struct {
|
||||
lex *lexer.Lexer
|
||||
cur lexer.Token
|
||||
next lexer.Token
|
||||
}
|
||||
|
||||
func newParser(input string) *parser {
|
||||
p := &parser{lex: lexer.New(input)}
|
||||
p.next = p.lex.Next()
|
||||
p.advance()
|
||||
return p
|
||||
}
|
||||
|
||||
func (p *parser) advance() {
|
||||
p.cur = p.next
|
||||
p.next = p.lex.Next()
|
||||
}
|
||||
|
||||
func (p *parser) parseQuery() (*ast.Query, error) {
|
||||
base, err := p.parseBoolExpr()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
q := &ast.Query{Base: base}
|
||||
for p.cur.Kind == lexer.Pipe {
|
||||
p.advance()
|
||||
stage, err := p.parsePipeStage()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
q.Pipes = append(q.Pipes, stage)
|
||||
}
|
||||
if p.cur.Kind != lexer.EOF {
|
||||
return nil, p.errorf("unexpected %s after query", p.cur)
|
||||
}
|
||||
return q, nil
|
||||
}
|
||||
|
||||
func (p *parser) parseBoolExpr() (ast.BoolExpr, error) {
|
||||
var expr ast.BoolExpr
|
||||
term, err := p.parseTerm()
|
||||
if err != nil {
|
||||
return expr, err
|
||||
}
|
||||
expr.Terms = append(expr.Terms, term)
|
||||
|
||||
for {
|
||||
var conj string
|
||||
switch {
|
||||
case p.cur.Kind == lexer.Ident && (p.cur.Value == "and" || p.cur.Value == "or"):
|
||||
conj = p.cur.Value
|
||||
p.advance()
|
||||
case p.canStartTerm():
|
||||
// Adjacent bare terms with no explicit keyword between them
|
||||
// implicitly AND, matching SPL's convention (e.g. `error
|
||||
// timeout` means `error AND timeout`).
|
||||
conj = "and"
|
||||
default:
|
||||
return expr, nil
|
||||
}
|
||||
term, err := p.parseTerm()
|
||||
if err != nil {
|
||||
return expr, err
|
||||
}
|
||||
expr.Terms = append(expr.Terms, term)
|
||||
expr.Conjs = append(expr.Conjs, conj)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *parser) canStartTerm() bool {
|
||||
switch p.cur.Kind {
|
||||
case lexer.Ident, lexer.String:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (p *parser) parseTerm() (ast.Term, error) {
|
||||
switch p.cur.Kind {
|
||||
case lexer.Ident:
|
||||
if p.cur.Value == "earliest" || p.cur.Value == "latest" {
|
||||
return p.parseTimeBound()
|
||||
}
|
||||
if p.cur.Value == "message" && p.next.Kind == lexer.Colon {
|
||||
return p.parseExplicitFreeText()
|
||||
}
|
||||
if isComparatorStart(p.next.Kind) {
|
||||
return p.parseComparison()
|
||||
}
|
||||
// A bare word with no comparator following it is a free-text
|
||||
// search term, not a malformed comparison.
|
||||
val := p.cur.Value
|
||||
p.advance()
|
||||
return ast.FreeText{Query: val}, nil
|
||||
case lexer.String:
|
||||
val := p.cur.Value
|
||||
p.advance()
|
||||
return ast.FreeText{Query: val}, nil
|
||||
default:
|
||||
return nil, p.errorf("expected a filter, comparison, or search term, got %s", p.cur)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *parser) parseComparison() (ast.Term, error) {
|
||||
field := p.cur.Value
|
||||
p.advance()
|
||||
op, err := p.parseComparator()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value, err := p.parseValue()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ast.Comparison{Field: field, Op: op, Value: value}, nil
|
||||
}
|
||||
|
||||
func (p *parser) parseComparator() (string, error) {
|
||||
if !isComparatorStart(p.cur.Kind) {
|
||||
return "", p.errorf("expected a comparator (=, !=, >, >=, <, <=), got %s", p.cur)
|
||||
}
|
||||
op := p.cur.Value
|
||||
p.advance()
|
||||
return op, nil
|
||||
}
|
||||
|
||||
func isComparatorStart(k lexer.Kind) bool {
|
||||
switch k {
|
||||
case lexer.Eq, lexer.Neq, lexer.Gt, lexer.Gte, lexer.Lt, lexer.Lte:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (p *parser) parseValue() (string, error) {
|
||||
switch p.cur.Kind {
|
||||
case lexer.Ident, lexer.String, lexer.Number:
|
||||
v := p.cur.Value
|
||||
p.advance()
|
||||
return v, nil
|
||||
default:
|
||||
return "", p.errorf("expected a value, got %s", p.cur)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *parser) parseTimeBound() (ast.Term, error) {
|
||||
kind := p.cur.Value
|
||||
p.advance()
|
||||
if err := p.expect(lexer.Eq); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
expr, err := p.parseTimeExpr()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ast.TimeBound{Kind: kind, Expr: expr}, nil
|
||||
}
|
||||
|
||||
func (p *parser) parseTimeExpr() (ast.TimeExpr, error) {
|
||||
if p.cur.Kind == lexer.String {
|
||||
v := p.cur.Value
|
||||
p.advance()
|
||||
return ast.TimeExpr{Absolute: v}, nil
|
||||
}
|
||||
|
||||
sign := 1
|
||||
switch p.cur.Kind {
|
||||
case lexer.Minus:
|
||||
sign = -1
|
||||
p.advance()
|
||||
case lexer.Plus:
|
||||
p.advance()
|
||||
}
|
||||
|
||||
if p.cur.Kind != lexer.Number {
|
||||
return ast.TimeExpr{}, p.errorf("expected a quoted absolute timestamp or a relative offset like -1h, got %s", p.cur)
|
||||
}
|
||||
n, err := strconv.Atoi(p.cur.Value)
|
||||
if err != nil {
|
||||
return ast.TimeExpr{}, p.errorf("invalid number %q in time expression", p.cur.Value)
|
||||
}
|
||||
p.advance()
|
||||
|
||||
if p.cur.Kind != lexer.Ident || !isValidTimeUnit(p.cur.Value) {
|
||||
return ast.TimeExpr{}, p.errorf("expected a time unit (s/m/h/d/w) after %d, got %s", n, p.cur)
|
||||
}
|
||||
unit := p.cur.Value
|
||||
p.advance()
|
||||
|
||||
return ast.TimeExpr{IsRelative: true, RelativeSign: sign, RelativeN: n, RelativeUnit: unit}, nil
|
||||
}
|
||||
|
||||
func isValidTimeUnit(u string) bool {
|
||||
switch u {
|
||||
case "s", "m", "h", "d", "w":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (p *parser) parseExplicitFreeText() (ast.Term, error) {
|
||||
p.advance() // "message"
|
||||
if err := p.expect(lexer.Colon); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if p.cur.Kind != lexer.String {
|
||||
return nil, p.errorf("expected a quoted string after message:, got %s", p.cur)
|
||||
}
|
||||
v := p.cur.Value
|
||||
p.advance()
|
||||
return ast.FreeText{Query: v}, nil
|
||||
}
|
||||
|
||||
func (p *parser) parsePipeStage() (ast.PipeStage, error) {
|
||||
if p.cur.Kind != lexer.Ident {
|
||||
return nil, p.errorf("expected a pipe stage (where/stats/sort/fields/head/tail), got %s", p.cur)
|
||||
}
|
||||
switch p.cur.Value {
|
||||
case "where":
|
||||
p.advance()
|
||||
expr, err := p.parseBoolExpr()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ast.WhereStage{Expr: expr}, nil
|
||||
case "stats":
|
||||
return p.parseStatsStage()
|
||||
case "sort":
|
||||
return p.parseSortStage()
|
||||
case "fields":
|
||||
return p.parseFieldsStage()
|
||||
case "head":
|
||||
return p.parseHeadTailStage(false)
|
||||
case "tail":
|
||||
return p.parseHeadTailStage(true)
|
||||
default:
|
||||
return nil, p.errorf("unknown pipe stage %q (expected where/stats/sort/fields/head/tail)", p.cur.Value)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *parser) parseStatsStage() (ast.PipeStage, error) {
|
||||
p.advance() // "stats"
|
||||
var stage ast.StatsStage
|
||||
|
||||
agg, err := p.parseAggCall()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stage.Aggs = append(stage.Aggs, agg)
|
||||
|
||||
for p.cur.Kind == lexer.Comma {
|
||||
p.advance()
|
||||
agg, err := p.parseAggCall()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stage.Aggs = append(stage.Aggs, agg)
|
||||
}
|
||||
|
||||
if p.cur.Kind == lexer.Ident && p.cur.Value == "by" {
|
||||
p.advance()
|
||||
field, err := p.parseFieldIdent()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stage.By = append(stage.By, field)
|
||||
for p.cur.Kind == lexer.Comma {
|
||||
p.advance()
|
||||
field, err := p.parseFieldIdent()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stage.By = append(stage.By, field)
|
||||
}
|
||||
}
|
||||
|
||||
return stage, nil
|
||||
}
|
||||
|
||||
func (p *parser) parseAggCall() (ast.AggCall, error) {
|
||||
if p.cur.Kind != lexer.Ident {
|
||||
return ast.AggCall{}, p.errorf("expected an aggregation function (count/sum/avg/min/max), got %s", p.cur)
|
||||
}
|
||||
fn := p.cur.Value
|
||||
if !isValidAggFunc(fn) {
|
||||
return ast.AggCall{}, p.errorf("unknown aggregation function %q (want count/sum/avg/min/max)", fn)
|
||||
}
|
||||
p.advance()
|
||||
|
||||
// Parens are optional when there's no field: `count`, `count()`, and
|
||||
// `count(*)` are all equivalent. `sum(field)` etc. still need them,
|
||||
// since that's the only way to name the field.
|
||||
var field string
|
||||
if p.cur.Kind == lexer.LParen {
|
||||
p.advance()
|
||||
switch p.cur.Kind {
|
||||
case lexer.Ident:
|
||||
field = p.cur.Value
|
||||
p.advance()
|
||||
case lexer.Star:
|
||||
p.advance() // count(*) is the same as count()
|
||||
}
|
||||
if err := p.expect(lexer.RParen); err != nil {
|
||||
return ast.AggCall{}, err
|
||||
}
|
||||
}
|
||||
|
||||
var alias string
|
||||
if p.cur.Kind == lexer.Ident && p.cur.Value == "as" {
|
||||
p.advance()
|
||||
if p.cur.Kind != lexer.Ident {
|
||||
return ast.AggCall{}, p.errorf("expected an alias after 'as', got %s", p.cur)
|
||||
}
|
||||
alias = p.cur.Value
|
||||
p.advance()
|
||||
}
|
||||
|
||||
return ast.AggCall{Func: fn, Field: field, Alias: alias}, nil
|
||||
}
|
||||
|
||||
func isValidAggFunc(f string) bool {
|
||||
switch f {
|
||||
case "count", "sum", "avg", "min", "max":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (p *parser) parseSortStage() (ast.PipeStage, error) {
|
||||
p.advance() // "sort"
|
||||
var stage ast.SortStage
|
||||
|
||||
field, err := p.parseSortField()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stage.Fields = append(stage.Fields, field)
|
||||
|
||||
for p.cur.Kind == lexer.Comma {
|
||||
p.advance()
|
||||
field, err := p.parseSortField()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stage.Fields = append(stage.Fields, field)
|
||||
}
|
||||
return stage, nil
|
||||
}
|
||||
|
||||
func (p *parser) parseSortField() (ast.SortField, error) {
|
||||
desc := true // no explicit sign defaults to descending, same as an explicit "-"
|
||||
switch p.cur.Kind {
|
||||
case lexer.Minus:
|
||||
p.advance()
|
||||
case lexer.Plus:
|
||||
desc = false
|
||||
p.advance()
|
||||
}
|
||||
if p.cur.Kind != lexer.Ident {
|
||||
return ast.SortField{}, p.errorf("expected a field name in sort, got %s", p.cur)
|
||||
}
|
||||
field := p.cur.Value
|
||||
p.advance()
|
||||
return ast.SortField{Field: field, Desc: desc}, nil
|
||||
}
|
||||
|
||||
func (p *parser) parseFieldsStage() (ast.PipeStage, error) {
|
||||
p.advance() // "fields"
|
||||
var stage ast.FieldsStage
|
||||
field, err := p.parseFieldIdent()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stage.Fields = append(stage.Fields, field)
|
||||
for p.cur.Kind == lexer.Comma {
|
||||
p.advance()
|
||||
field, err := p.parseFieldIdent()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stage.Fields = append(stage.Fields, field)
|
||||
}
|
||||
return stage, nil
|
||||
}
|
||||
|
||||
func (p *parser) parseFieldIdent() (string, error) {
|
||||
if p.cur.Kind != lexer.Ident {
|
||||
return "", p.errorf("expected a field name, got %s", p.cur)
|
||||
}
|
||||
v := p.cur.Value
|
||||
p.advance()
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func (p *parser) parseHeadTailStage(tail bool) (ast.PipeStage, error) {
|
||||
p.advance() // "head" / "tail"
|
||||
if p.cur.Kind == lexer.Number {
|
||||
n, err := strconv.Atoi(p.cur.Value)
|
||||
if err != nil {
|
||||
return nil, p.errorf("invalid number %q", p.cur.Value)
|
||||
}
|
||||
p.advance()
|
||||
if tail {
|
||||
return ast.TailStage{N: n, HasN: true}, nil
|
||||
}
|
||||
return ast.HeadStage{N: n, HasN: true}, nil
|
||||
}
|
||||
if tail {
|
||||
return ast.TailStage{}, nil
|
||||
}
|
||||
return ast.HeadStage{}, nil
|
||||
}
|
||||
|
||||
func (p *parser) expect(k lexer.Kind) error {
|
||||
if p.cur.Kind != k {
|
||||
return p.errorf("expected %s, got %s", k, p.cur)
|
||||
}
|
||||
p.advance()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *parser) errorf(format string, args ...any) error {
|
||||
return fmt.Errorf("query syntax error at position %d: %s", p.cur.Pos, fmt.Sprintf(format, args...))
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/sentry/sentry/api/internal/querylang/ast"
|
||||
)
|
||||
|
||||
func TestParseSimpleFilter(t *testing.T) {
|
||||
q, err := Parse(`service=api`)
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
if len(q.Base.Terms) != 1 {
|
||||
t.Fatalf("expected 1 base term, got %d", len(q.Base.Terms))
|
||||
}
|
||||
cmp, ok := q.Base.Terms[0].(ast.Comparison)
|
||||
if !ok {
|
||||
t.Fatalf("expected Comparison, got %T", q.Base.Terms[0])
|
||||
}
|
||||
if cmp.Field != "service" || cmp.Op != "=" || cmp.Value != "api" {
|
||||
t.Fatalf("unexpected comparison: %+v", cmp)
|
||||
}
|
||||
if len(q.Pipes) != 0 {
|
||||
t.Fatalf("expected no pipes, got %d", len(q.Pipes))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFullPipeline(t *testing.T) {
|
||||
q, err := Parse(`service=api | where status>=500 | stats count(*) as errors by host | sort -errors`)
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
if len(q.Pipes) != 3 {
|
||||
t.Fatalf("expected 3 pipe stages, got %d: %+v", len(q.Pipes), q.Pipes)
|
||||
}
|
||||
|
||||
where, ok := q.Pipes[0].(ast.WhereStage)
|
||||
if !ok {
|
||||
t.Fatalf("stage 0: expected WhereStage, got %T", q.Pipes[0])
|
||||
}
|
||||
cmp := where.Expr.Terms[0].(ast.Comparison)
|
||||
if cmp.Field != "status" || cmp.Op != ">=" || cmp.Value != "500" {
|
||||
t.Fatalf("unexpected where comparison: %+v", cmp)
|
||||
}
|
||||
|
||||
stats, ok := q.Pipes[1].(ast.StatsStage)
|
||||
if !ok {
|
||||
t.Fatalf("stage 1: expected StatsStage, got %T", q.Pipes[1])
|
||||
}
|
||||
if len(stats.Aggs) != 1 || stats.Aggs[0].Func != "count" || stats.Aggs[0].Alias != "errors" {
|
||||
t.Fatalf("unexpected stats aggs: %+v", stats.Aggs)
|
||||
}
|
||||
if len(stats.By) != 1 || stats.By[0] != "host" {
|
||||
t.Fatalf("unexpected stats by: %+v", stats.By)
|
||||
}
|
||||
|
||||
sort, ok := q.Pipes[2].(ast.SortStage)
|
||||
if !ok {
|
||||
t.Fatalf("stage 2: expected SortStage, got %T", q.Pipes[2])
|
||||
}
|
||||
if len(sort.Fields) != 1 || sort.Fields[0].Field != "errors" || !sort.Fields[0].Desc {
|
||||
t.Fatalf("unexpected sort fields: %+v", sort.Fields)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseExplicitFreeTextWithAggregation(t *testing.T) {
|
||||
q, err := Parse(`message:"connection refused" | stats count by host`)
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
ft, ok := q.Base.Terms[0].(ast.FreeText)
|
||||
if !ok {
|
||||
t.Fatalf("expected FreeText, got %T", q.Base.Terms[0])
|
||||
}
|
||||
if ft.Query != "connection refused" {
|
||||
t.Fatalf("Query = %q, want %q", ft.Query, "connection refused")
|
||||
}
|
||||
stats := q.Pipes[0].(ast.StatsStage)
|
||||
if stats.Aggs[0].Func != "count" || stats.Aggs[0].Field != "" {
|
||||
t.Fatalf("unexpected agg: %+v", stats.Aggs[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseBareWordIsFreeText(t *testing.T) {
|
||||
q, err := Parse(`timeout`)
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
ft, ok := q.Base.Terms[0].(ast.FreeText)
|
||||
if !ok || ft.Query != "timeout" {
|
||||
t.Fatalf("expected FreeText(timeout), got %+v", q.Base.Terms[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseImplicitAndBetweenBareTerms(t *testing.T) {
|
||||
q, err := Parse(`error timeout`)
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
if len(q.Base.Terms) != 2 {
|
||||
t.Fatalf("expected 2 terms, got %d", len(q.Base.Terms))
|
||||
}
|
||||
if len(q.Base.Conjs) != 1 || q.Base.Conjs[0] != "and" {
|
||||
t.Fatalf("expected implicit 'and', got %+v", q.Base.Conjs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseExplicitAndOr(t *testing.T) {
|
||||
q, err := Parse(`service=api and status=500`)
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
if len(q.Base.Conjs) != 1 || q.Base.Conjs[0] != "and" {
|
||||
t.Fatalf("expected explicit 'and', got %+v", q.Base.Conjs)
|
||||
}
|
||||
|
||||
q2, err := Parse(`service=api or service=web`)
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
if len(q2.Base.Conjs) != 1 || q2.Base.Conjs[0] != "or" {
|
||||
t.Fatalf("expected 'or', got %+v", q2.Base.Conjs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTimeBoundsRelativeAndAbsolute(t *testing.T) {
|
||||
q, err := Parse(`earliest=-1h latest="2026-08-14T00:00:00Z"`)
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
if len(q.Base.Terms) != 2 {
|
||||
t.Fatalf("expected 2 terms, got %d", len(q.Base.Terms))
|
||||
}
|
||||
earliest := q.Base.Terms[0].(ast.TimeBound)
|
||||
if earliest.Kind != "earliest" || !earliest.Expr.IsRelative || earliest.Expr.RelativeSign != -1 ||
|
||||
earliest.Expr.RelativeN != 1 || earliest.Expr.RelativeUnit != "h" {
|
||||
t.Fatalf("unexpected earliest: %+v", earliest)
|
||||
}
|
||||
latest := q.Base.Terms[1].(ast.TimeBound)
|
||||
if latest.Kind != "latest" || latest.Expr.Absolute != "2026-08-14T00:00:00Z" {
|
||||
t.Fatalf("unexpected latest: %+v", latest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFieldsStage(t *testing.T) {
|
||||
q, err := Parse(`service=api | fields host, message, severity`)
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
fields := q.Pipes[0].(ast.FieldsStage)
|
||||
want := []string{"host", "message", "severity"}
|
||||
if len(fields.Fields) != len(want) {
|
||||
t.Fatalf("Fields = %v, want %v", fields.Fields, want)
|
||||
}
|
||||
for i := range want {
|
||||
if fields.Fields[i] != want[i] {
|
||||
t.Fatalf("Fields = %v, want %v", fields.Fields, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseHeadTailWithAndWithoutN(t *testing.T) {
|
||||
q, err := Parse(`service=api | head 10`)
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
head := q.Pipes[0].(ast.HeadStage)
|
||||
if !head.HasN || head.N != 10 {
|
||||
t.Fatalf("unexpected head: %+v", head)
|
||||
}
|
||||
|
||||
q2, err := Parse(`service=api | tail`)
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
tail := q2.Pipes[0].(ast.TailStage)
|
||||
if tail.HasN {
|
||||
t.Fatalf("expected no N, got %+v", tail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDottedFieldName(t *testing.T) {
|
||||
q, err := Parse(`winevt.event_id=4625`)
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
cmp := q.Base.Terms[0].(ast.Comparison)
|
||||
if cmp.Field != "winevt.event_id" || cmp.Value != "4625" {
|
||||
t.Fatalf("unexpected comparison: %+v", cmp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSortAscendingWithPlus(t *testing.T) {
|
||||
q, err := Parse(`service=api | sort +host`)
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
sort := q.Pipes[0].(ast.SortStage)
|
||||
if sort.Fields[0].Desc {
|
||||
t.Fatalf("expected ascending sort, got %+v", sort.Fields[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMultipleSortFields(t *testing.T) {
|
||||
q, err := Parse(`service=api | sort -severity, +host`)
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
sort := q.Pipes[0].(ast.SortStage)
|
||||
if len(sort.Fields) != 2 {
|
||||
t.Fatalf("expected 2 sort fields, got %d", len(sort.Fields))
|
||||
}
|
||||
if sort.Fields[0].Field != "severity" || !sort.Fields[0].Desc {
|
||||
t.Fatalf("unexpected first sort field: %+v", sort.Fields[0])
|
||||
}
|
||||
if sort.Fields[1].Field != "host" || sort.Fields[1].Desc {
|
||||
t.Fatalf("unexpected second sort field: %+v", sort.Fields[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMultipleAggregations(t *testing.T) {
|
||||
q, err := Parse(`service=api | stats count() as n, avg(latency_ms) as avg_latency by host`)
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
stats := q.Pipes[0].(ast.StatsStage)
|
||||
if len(stats.Aggs) != 2 {
|
||||
t.Fatalf("expected 2 aggs, got %d: %+v", len(stats.Aggs), stats.Aggs)
|
||||
}
|
||||
if stats.Aggs[1].Func != "avg" || stats.Aggs[1].Field != "latency_ms" || stats.Aggs[1].Alias != "avg_latency" {
|
||||
t.Fatalf("unexpected second agg: %+v", stats.Aggs[1])
|
||||
}
|
||||
}
|
||||
|
||||
// --- error cases ---
|
||||
|
||||
func TestParseErrorEmptyQuery(t *testing.T) {
|
||||
if _, err := Parse(``); err == nil {
|
||||
t.Fatal("expected an error for an empty query")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseErrorUnknownPipeStage(t *testing.T) {
|
||||
if _, err := Parse(`service=api | bogus`); err == nil {
|
||||
t.Fatal("expected an error for an unknown pipe stage")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseErrorMissingComparatorValue(t *testing.T) {
|
||||
if _, err := Parse(`service=`); err == nil {
|
||||
t.Fatal("expected an error for a missing comparison value")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseErrorUnknownAggFunc(t *testing.T) {
|
||||
if _, err := Parse(`service=api | stats median(latency)`); err == nil {
|
||||
t.Fatal("expected an error for an unknown aggregation function")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseErrorInvalidTimeUnit(t *testing.T) {
|
||||
if _, err := Parse(`earliest=-1x`); err == nil {
|
||||
t.Fatal("expected an error for an invalid time unit")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseErrorTrailingGarbage(t *testing.T) {
|
||||
if _, err := Parse(`service=api extra ) tokens`); err == nil {
|
||||
t.Fatal("expected an error for trailing unparseable tokens")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseErrorUnterminatedStringInFreeText(t *testing.T) {
|
||||
if _, err := Parse(`message:"unterminated`); err == nil {
|
||||
t.Fatal("expected an error for an unterminated quoted string")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,286 @@
|
||||
// Package planner compiles a query string (either syntax) into ir.Plan.
|
||||
// This is the single entry point querylang exposes to callers (the /query
|
||||
// HTTP handler) -- see Compile.
|
||||
package planner
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/sentry/sentry/api/internal/querylang/ast"
|
||||
"github.com/sentry/sentry/api/internal/querylang/ir"
|
||||
"github.com/sentry/sentry/api/internal/querylang/parser"
|
||||
)
|
||||
|
||||
// Language selects which syntax a query is written in.
|
||||
type Language string
|
||||
|
||||
const (
|
||||
Auto Language = "" // detect from the query text (default)
|
||||
SQL Language = "sql"
|
||||
SPL Language = "spl" // the pipe syntax; named to match the query-language-reference doc
|
||||
)
|
||||
|
||||
const defaultLimit = 100
|
||||
|
||||
// Compile turns a query string into a Plan. language overrides
|
||||
// auto-detection; pass Auto to use the SELECT-prefix heuristic (see
|
||||
// /docs/query-language-design.md's "Detection" section) -- this exists
|
||||
// for the rare case a pipe query legitimately starts with the literal
|
||||
// word "select" as a bare search term.
|
||||
func Compile(query string, language Language, now time.Time) (*ir.Plan, error) {
|
||||
isSQL := language == SQL
|
||||
if language == Auto {
|
||||
isSQL = looksLikeSQL(query)
|
||||
}
|
||||
|
||||
if isSQL {
|
||||
if err := validateSelectOnly(query); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ir.Plan{RawSQL: strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(query), ";"))}, nil
|
||||
}
|
||||
|
||||
q, err := parser.Parse(query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return compileQuery(q, now)
|
||||
}
|
||||
|
||||
func looksLikeSQL(query string) bool {
|
||||
trimmed := strings.TrimSpace(query)
|
||||
if trimmed == "" {
|
||||
return false
|
||||
}
|
||||
fields := strings.Fields(trimmed)
|
||||
return len(fields) > 0 && strings.EqualFold(fields[0], "SELECT")
|
||||
}
|
||||
|
||||
func compileQuery(q *ast.Query, now time.Time) (*ir.Plan, error) {
|
||||
plan := &ir.Plan{}
|
||||
|
||||
textParts, err := compileBoolExpr(q.Base, plan, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, stage := range q.Pipes {
|
||||
switch s := stage.(type) {
|
||||
case ast.WhereStage:
|
||||
parts, err := compileBoolExpr(s.Expr, plan, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
textParts = append(textParts, parts...)
|
||||
case ast.StatsStage:
|
||||
agg, err := compileStats(s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plan.Aggregation = agg
|
||||
case ast.SortStage:
|
||||
for _, f := range s.Fields {
|
||||
plan.Sort = append(plan.Sort, ir.SortField{Field: f.Field, Desc: f.Desc})
|
||||
}
|
||||
case ast.FieldsStage:
|
||||
plan.Fields = append(plan.Fields, s.Fields...)
|
||||
case ast.HeadStage:
|
||||
n := defaultLimit
|
||||
if s.HasN {
|
||||
n = s.N
|
||||
}
|
||||
plan.Limit = &ir.Limit{N: n, Tail: false}
|
||||
case ast.TailStage:
|
||||
n := defaultLimit
|
||||
if s.HasN {
|
||||
n = s.N
|
||||
}
|
||||
plan.Limit = &ir.Limit{N: n, Tail: true}
|
||||
default:
|
||||
return nil, fmt.Errorf("internal error: unhandled pipe stage %T", stage)
|
||||
}
|
||||
}
|
||||
|
||||
if len(textParts) > 0 {
|
||||
plan.TextSearch = []ir.TextPredicate{{Query: strings.Join(textParts, " ")}}
|
||||
}
|
||||
|
||||
return plan, nil
|
||||
}
|
||||
|
||||
// textPart is one piece of a combined Tantivy query string, tagged with
|
||||
// the conjunction that precedes it (empty for the first piece).
|
||||
type textPart struct {
|
||||
conj string // "", "and", "or"
|
||||
query string
|
||||
}
|
||||
|
||||
// compileBoolExpr walks one bool_expr (the base search, or a `where`
|
||||
// stage's expression), populating plan.Filters and plan.TimeRange
|
||||
// directly, and returning free-text pieces for the caller to fold into
|
||||
// the combined Tantivy query string.
|
||||
//
|
||||
// Scope decision: "or" is only supported between free-text terms, which
|
||||
// Tantivy's own query parser handles natively once composed into one
|
||||
// string. "or" between structured comparisons/time-bounds is rejected
|
||||
// with a clear error rather than silently compiled as "and" -- see
|
||||
// /docs/query-language-reference.md's limitations section. This keeps
|
||||
// the executor's generated SQL a flat AND-only WHERE clause, which is
|
||||
// most of what real queries need; full boolean-tree support for
|
||||
// structured filters is future work if usage shows it's needed.
|
||||
func compileBoolExpr(expr ast.BoolExpr, plan *ir.Plan, now time.Time) ([]string, error) {
|
||||
var textParts []string
|
||||
|
||||
for i, term := range expr.Terms {
|
||||
conj := ""
|
||||
if i > 0 {
|
||||
conj = expr.Conjs[i-1]
|
||||
}
|
||||
|
||||
switch t := term.(type) {
|
||||
case ast.Comparison:
|
||||
if conj == "or" {
|
||||
return nil, fmt.Errorf("query error: \"or\" is not supported between structured filters (%q) in Phase 2 -- only between free-text search terms", t.Field)
|
||||
}
|
||||
plan.Filters = append(plan.Filters, ir.FilterPredicate{Field: t.Field, Op: t.Op, Value: t.Value})
|
||||
case ast.TimeBound:
|
||||
if conj == "or" {
|
||||
return nil, fmt.Errorf("query error: \"or\" is not supported on time bounds (%s) in Phase 2", t.Kind)
|
||||
}
|
||||
if err := applyTimeBound(plan, t, now); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
case ast.FreeText:
|
||||
q := t.Query
|
||||
if strings.ContainsAny(q, " \t") {
|
||||
q = `"` + strings.ReplaceAll(q, `"`, `\"`) + `"`
|
||||
}
|
||||
if conj == "or" {
|
||||
textParts = append(textParts, "OR", q)
|
||||
} else if len(textParts) > 0 {
|
||||
textParts = append(textParts, "AND", q)
|
||||
} else {
|
||||
textParts = append(textParts, q)
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("internal error: unhandled term %T", term)
|
||||
}
|
||||
}
|
||||
|
||||
return textParts, nil
|
||||
}
|
||||
|
||||
func applyTimeBound(plan *ir.Plan, t ast.TimeBound, now time.Time) error {
|
||||
when, err := resolveTimeExpr(t.Expr, now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if plan.TimeRange == nil {
|
||||
plan.TimeRange = &ir.TimeRange{}
|
||||
}
|
||||
switch t.Kind {
|
||||
case "earliest":
|
||||
plan.TimeRange.From = when
|
||||
case "latest":
|
||||
plan.TimeRange.To = when
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func resolveTimeExpr(e ast.TimeExpr, now time.Time) (time.Time, error) {
|
||||
if !e.IsRelative {
|
||||
t, err := time.Parse(time.RFC3339, e.Absolute)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("query error: invalid absolute timestamp %q, want RFC3339 (e.g. 2026-08-14T00:00:00Z): %w", e.Absolute, err)
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
var d time.Duration
|
||||
switch e.RelativeUnit {
|
||||
case "s":
|
||||
d = time.Duration(e.RelativeN) * time.Second
|
||||
case "m":
|
||||
d = time.Duration(e.RelativeN) * time.Minute
|
||||
case "h":
|
||||
d = time.Duration(e.RelativeN) * time.Hour
|
||||
case "d":
|
||||
d = time.Duration(e.RelativeN) * 24 * time.Hour
|
||||
case "w":
|
||||
d = time.Duration(e.RelativeN) * 7 * 24 * time.Hour
|
||||
default:
|
||||
return time.Time{}, fmt.Errorf("internal error: unknown time unit %q", e.RelativeUnit)
|
||||
}
|
||||
if e.RelativeSign < 0 {
|
||||
d = -d
|
||||
}
|
||||
return now.Add(d), nil
|
||||
}
|
||||
|
||||
func compileStats(s ast.StatsStage) (*ir.Aggregation, error) {
|
||||
agg := &ir.Aggregation{GroupBy: s.By}
|
||||
seen := map[string]bool{}
|
||||
for _, a := range s.Aggs {
|
||||
if a.Func != "count" && a.Field == "" {
|
||||
return nil, fmt.Errorf("query error: %s() requires a field, e.g. %s(latency_ms)", a.Func, a.Func)
|
||||
}
|
||||
alias := a.Alias
|
||||
if alias == "" {
|
||||
alias = defaultAggAlias(a)
|
||||
}
|
||||
if seen[alias] && a.Alias == "" {
|
||||
// Two unnamed aggs of the same shape would otherwise collide
|
||||
// (e.g. `stats sum(a), sum(b)` both defaulting to "sum") --
|
||||
// disambiguate by field name.
|
||||
alias = alias + "_" + a.Field
|
||||
}
|
||||
seen[alias] = true
|
||||
agg.Funcs = append(agg.Funcs, ir.AggFunc{Func: a.Func, Field: a.Field, Alias: alias})
|
||||
}
|
||||
return agg, nil
|
||||
}
|
||||
|
||||
func defaultAggAlias(a ast.AggCall) string {
|
||||
if a.Func == "count" {
|
||||
return "count"
|
||||
}
|
||||
return a.Func
|
||||
}
|
||||
|
||||
// --- SQL escape hatch validation, ported from the Phase 0/1
|
||||
// api/internal/queryapi/validate.go guard it replaces (see task 4) ---
|
||||
|
||||
// disallowedKeyword is defense-in-depth on top of the SELECT-only gate:
|
||||
// it catches mutating/administrative statements appearing anywhere in
|
||||
// the query, not just at the start. Word-boundary matching, not a real
|
||||
// SQL parser -- same tradeoffs as the Phase 0/1 version this replaces.
|
||||
var disallowedKeyword = regexp.MustCompile(`(?i)\b(insert|update|delete|alter|drop|truncate|create|grant|revoke|attach|detach|rename|kill|optimize|system|set|exchange|watch)\b`)
|
||||
|
||||
func validateSelectOnly(sql string) error {
|
||||
trimmed := strings.TrimSpace(sql)
|
||||
if trimmed == "" {
|
||||
return fmt.Errorf("query must not be empty")
|
||||
}
|
||||
|
||||
trimmed = strings.TrimSpace(strings.TrimSuffix(trimmed, ";"))
|
||||
if trimmed == "" {
|
||||
return fmt.Errorf("query must not be empty")
|
||||
}
|
||||
if strings.Contains(trimmed, ";") {
|
||||
return fmt.Errorf("only a single statement is allowed")
|
||||
}
|
||||
|
||||
firstWord := strings.ToUpper(strings.Fields(trimmed)[0])
|
||||
if firstWord != "SELECT" {
|
||||
return fmt.Errorf("only SELECT queries are allowed")
|
||||
}
|
||||
|
||||
if disallowedKeyword.MatchString(trimmed) {
|
||||
return fmt.Errorf("query contains a disallowed keyword")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
package planner
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/sentry/sentry/api/internal/querylang/ir"
|
||||
)
|
||||
|
||||
var fixedNow = time.Date(2026, 8, 14, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
func TestCompileDetectsSQL(t *testing.T) {
|
||||
plan, err := Compile(`SELECT * FROM logs LIMIT 10`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if plan.RawSQL == "" {
|
||||
t.Fatalf("expected RawSQL to be set, got plan: %+v", plan)
|
||||
}
|
||||
if plan.RawSQL != "SELECT * FROM logs LIMIT 10" {
|
||||
t.Fatalf("RawSQL = %q", plan.RawSQL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileDetectsSQLCaseInsensitive(t *testing.T) {
|
||||
plan, err := Compile(`select 1`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if plan.RawSQL != "select 1" {
|
||||
t.Fatalf("RawSQL = %q", plan.RawSQL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileRejectsNonSelectSQLKeyword(t *testing.T) {
|
||||
_, err := Compile(`DELETE FROM logs`, SQL, fixedNow)
|
||||
if err == nil {
|
||||
t.Fatal("expected an error for a non-SELECT statement forced to SQL language")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileExplicitLanguageOverridesAutoDetect(t *testing.T) {
|
||||
// "select" as a bare free-text search term -- would be misdetected
|
||||
// as SQL by the heuristic alone, hence the override.
|
||||
plan, err := Compile(`select`, SPL, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if plan.RawSQL != "" {
|
||||
t.Fatalf("expected pipe-syntax compilation, got RawSQL = %q", plan.RawSQL)
|
||||
}
|
||||
if len(plan.TextSearch) != 1 || plan.TextSearch[0].Query != "select" {
|
||||
t.Fatalf("expected a free-text search for 'select', got %+v", plan.TextSearch)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileSimpleFilter(t *testing.T) {
|
||||
plan, err := Compile(`service=api`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
want := ir.FilterPredicate{Field: "service", Op: "=", Value: "api"}
|
||||
if len(plan.Filters) != 1 || plan.Filters[0] != want {
|
||||
t.Fatalf("unexpected filters: %+v, want [%+v]", plan.Filters, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileFullPipeline(t *testing.T) {
|
||||
plan, err := Compile(`service=api | where status>=500 | stats count by host | sort -count`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if len(plan.Filters) != 2 {
|
||||
t.Fatalf("expected 2 filters (service=api, status>=500), got %+v", plan.Filters)
|
||||
}
|
||||
if plan.Aggregation == nil || len(plan.Aggregation.Funcs) != 1 || plan.Aggregation.Funcs[0].Alias != "count" {
|
||||
t.Fatalf("unexpected aggregation: %+v", plan.Aggregation)
|
||||
}
|
||||
if len(plan.Aggregation.GroupBy) != 1 || plan.Aggregation.GroupBy[0] != "host" {
|
||||
t.Fatalf("unexpected group by: %+v", plan.Aggregation.GroupBy)
|
||||
}
|
||||
if len(plan.Sort) != 1 || plan.Sort[0].Field != "count" || !plan.Sort[0].Desc {
|
||||
t.Fatalf("unexpected sort: %+v", plan.Sort)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileTextSearchWithAggregation(t *testing.T) {
|
||||
plan, err := Compile(`message:"connection refused" | stats count by host`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if len(plan.TextSearch) != 1 || plan.TextSearch[0].Query != `"connection refused"` {
|
||||
t.Fatalf("unexpected text search: %+v", plan.TextSearch)
|
||||
}
|
||||
if plan.Aggregation == nil {
|
||||
t.Fatal("expected an aggregation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileImplicitAndBetweenFreeTextTerms(t *testing.T) {
|
||||
plan, err := Compile(`error timeout`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if len(plan.TextSearch) != 1 {
|
||||
t.Fatalf("expected 1 combined text predicate, got %+v", plan.TextSearch)
|
||||
}
|
||||
if plan.TextSearch[0].Query != "error AND timeout" {
|
||||
t.Fatalf("Query = %q, want %q", plan.TextSearch[0].Query, "error AND timeout")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileOrBetweenFreeTextTerms(t *testing.T) {
|
||||
plan, err := Compile(`error or timeout`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if plan.TextSearch[0].Query != "error OR timeout" {
|
||||
t.Fatalf("Query = %q, want %q", plan.TextSearch[0].Query, "error OR timeout")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileOrBetweenStructuredFiltersErrors(t *testing.T) {
|
||||
_, err := Compile(`service=api or service=web`, Auto, fixedNow)
|
||||
if err == nil {
|
||||
t.Fatal("expected an error: OR between structured filters isn't supported in Phase 2")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "or") {
|
||||
t.Fatalf("error should mention 'or', got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileRelativeTimeBound(t *testing.T) {
|
||||
plan, err := Compile(`earliest=-1h`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if plan.TimeRange == nil {
|
||||
t.Fatal("expected a TimeRange")
|
||||
}
|
||||
want := fixedNow.Add(-1 * time.Hour)
|
||||
if !plan.TimeRange.From.Equal(want) {
|
||||
t.Fatalf("From = %v, want %v", plan.TimeRange.From, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileAbsoluteTimeBound(t *testing.T) {
|
||||
plan, err := Compile(`latest="2026-08-14T00:00:00Z"`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
want := time.Date(2026, 8, 14, 0, 0, 0, 0, time.UTC)
|
||||
if !plan.TimeRange.To.Equal(want) {
|
||||
t.Fatalf("To = %v, want %v", plan.TimeRange.To, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileInvalidAbsoluteTimestampErrors(t *testing.T) {
|
||||
_, err := Compile(`latest="not-a-timestamp"`, Auto, fixedNow)
|
||||
if err == nil {
|
||||
t.Fatal("expected an error for an invalid absolute timestamp")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileSumWithoutFieldErrors(t *testing.T) {
|
||||
_, err := Compile(`service=api | stats sum`, Auto, fixedNow)
|
||||
if err == nil {
|
||||
t.Fatal("expected an error: sum() requires a field")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileAggAliasDefaultsAndCollisionIsDisambiguated(t *testing.T) {
|
||||
plan, err := Compile(`service=api | stats sum(a), sum(b)`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if len(plan.Aggregation.Funcs) != 2 {
|
||||
t.Fatalf("expected 2 agg funcs, got %+v", plan.Aggregation.Funcs)
|
||||
}
|
||||
if plan.Aggregation.Funcs[0].Alias == plan.Aggregation.Funcs[1].Alias {
|
||||
t.Fatalf("expected distinct aliases, got both %q", plan.Aggregation.Funcs[0].Alias)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileHeadDefaultsLimit(t *testing.T) {
|
||||
plan, err := Compile(`service=api | head`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if plan.Limit == nil || plan.Limit.N != defaultLimit || plan.Limit.Tail {
|
||||
t.Fatalf("unexpected limit: %+v", plan.Limit)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileTailSetsTailFlag(t *testing.T) {
|
||||
plan, err := Compile(`service=api | tail 5`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if plan.Limit == nil || plan.Limit.N != 5 || !plan.Limit.Tail {
|
||||
t.Fatalf("unexpected limit: %+v", plan.Limit)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileFieldsProjection(t *testing.T) {
|
||||
plan, err := Compile(`service=api | fields host, message`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if len(plan.Fields) != 2 || plan.Fields[0] != "host" || plan.Fields[1] != "message" {
|
||||
t.Fatalf("unexpected fields: %+v", plan.Fields)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileParseErrorPropagates(t *testing.T) {
|
||||
_, err := Compile(`service=api | bogus`, Auto, fixedNow)
|
||||
if err == nil {
|
||||
t.Fatal("expected a parse error to propagate")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user