// Package parser is a hand-written recursive-descent parser for the // pipe-syntax query language, per /docs/query-language-design.md's // choice of parser approach (no combinator/generator library -- this // grammar is small and stable, and hand-written gives full control over // error messages, which matter for a user-facing query language). package parser import ( "fmt" "strconv" "github.com/cairnobs/cairnobs/api/internal/querylang/ast" "github.com/cairnobs/cairnobs/api/internal/querylang/lexer" ) // Parse parses a pipe-syntax query. Callers are responsible for routing // SQL (queries starting with "SELECT") elsewhere before calling this -- // see planner.Plan and /docs/query-language-design.md's "SQL escape // hatch" section for why this parser never sees SQL at all. func Parse(input string) (*ast.Query, error) { p := newParser(input) return p.parseQuery() } type parser struct { lex *lexer.Lexer cur lexer.Token next lexer.Token } func newParser(input string) *parser { p := &parser{lex: lexer.New(input)} p.next = p.lex.Next() p.advance() return p } func (p *parser) advance() { p.cur = p.next p.next = p.lex.Next() } func (p *parser) parseQuery() (*ast.Query, error) { q := &ast.Query{} // base_search is normally mandatory (see the grammar comment on // ast.Query), but every pipe-stage keyword is also a valid bare // Ident, so a query with no leading filter -- e.g. `stats count by // host`, documented as valid in // /docs/query-language-reference.md's "stats" section -- used to // get silently misparsed as a base_search of four ANDed free-text // terms ("stats", "count", "by", "host"), matching nothing rather // than erroring or aggregating. Recognizing a leading pipe-stage // keyword up front and skipping straight to stage-parsing (q.Base // stays its zero value; compileBoolExpr already treats zero Terms // as "match everything", so no planner/executor change is needed) // fixes that without a leading "|". atPipeStageKeyword's comparator // lookahead keeps a genuine field named e.g. "where" (`where=foo`) // parsing as a structured filter, same as today. if p.atPipeStageKeyword() { stage, err := p.parsePipeStage() if err != nil { return nil, err } q.Pipes = append(q.Pipes, stage) } else { base, err := p.parseBoolExpr() if err != nil { return nil, err } q.Base = base } for p.cur.Kind == lexer.Pipe { p.advance() stage, err := p.parsePipeStage() if err != nil { return nil, err } q.Pipes = append(q.Pipes, stage) } if p.cur.Kind != lexer.EOF { return nil, p.errorf("unexpected %s after query", p.cur) } return q, nil } // atPipeStageKeyword reports whether the parser is sitting on one of the // six pipe-stage keywords used as the query's very first token, with no // preceding base filter. The comparator lookahead disambiguates from a // structured comparison on a field that happens to share one of these // names (e.g. `where=foo`), which must keep parsing as a filter, not a // stage. func (p *parser) atPipeStageKeyword() bool { if p.cur.Kind != lexer.Ident || isComparatorStart(p.next.Kind) { return false } switch p.cur.Value { case "where", "stats", "sort", "fields", "head", "tail": return true default: return false } } func (p *parser) parseBoolExpr() (ast.BoolExpr, error) { var expr ast.BoolExpr term, err := p.parseTerm() if err != nil { return expr, err } expr.Terms = append(expr.Terms, term) for { var conj string switch { case p.cur.Kind == lexer.Ident && (p.cur.Value == "and" || p.cur.Value == "or"): conj = p.cur.Value p.advance() case p.canStartTerm(): // Adjacent bare terms with no explicit keyword between them // implicitly AND, matching SPL's convention (e.g. `error // timeout` means `error AND timeout`). conj = "and" default: return expr, nil } term, err := p.parseTerm() if err != nil { return expr, err } expr.Terms = append(expr.Terms, term) expr.Conjs = append(expr.Conjs, conj) } } func (p *parser) canStartTerm() bool { switch p.cur.Kind { case lexer.Ident, lexer.String: return true default: return false } } func (p *parser) parseTerm() (ast.Term, error) { switch p.cur.Kind { case lexer.Ident: if p.cur.Value == "earliest" || p.cur.Value == "latest" { return p.parseTimeBound() } if p.cur.Value == "message" && p.next.Kind == lexer.Colon { return p.parseExplicitFreeText() } if isComparatorStart(p.next.Kind) { return p.parseComparison() } // A bare word with no comparator following it is a free-text // search term, not a malformed comparison. val := p.cur.Value p.advance() return ast.FreeText{Query: val}, nil case lexer.String: val := p.cur.Value p.advance() return ast.FreeText{Query: val}, nil default: return nil, p.errorf("expected a filter, comparison, or search term, got %s", p.cur) } } func (p *parser) parseComparison() (ast.Term, error) { field := p.cur.Value p.advance() op, err := p.parseComparator() if err != nil { return nil, err } value, err := p.parseValue() if err != nil { return nil, err } return ast.Comparison{Field: field, Op: op, Value: value}, nil } func (p *parser) parseComparator() (string, error) { if !isComparatorStart(p.cur.Kind) { return "", p.errorf("expected a comparator (=, !=, >, >=, <, <=), got %s", p.cur) } op := p.cur.Value p.advance() return op, nil } func isComparatorStart(k lexer.Kind) bool { switch k { case lexer.Eq, lexer.Neq, lexer.Gt, lexer.Gte, lexer.Lt, lexer.Lte: return true default: return false } } func (p *parser) parseValue() (string, error) { switch p.cur.Kind { case lexer.Ident, lexer.String, lexer.Number: v := p.cur.Value p.advance() return v, nil default: return "", p.errorf("expected a value, got %s", p.cur) } } func (p *parser) parseTimeBound() (ast.Term, error) { kind := p.cur.Value p.advance() if err := p.expect(lexer.Eq); err != nil { return nil, err } expr, err := p.parseTimeExpr() if err != nil { return nil, err } return ast.TimeBound{Kind: kind, Expr: expr}, nil } func (p *parser) parseTimeExpr() (ast.TimeExpr, error) { if p.cur.Kind == lexer.String { v := p.cur.Value p.advance() return ast.TimeExpr{Absolute: v}, nil } sign := 1 switch p.cur.Kind { case lexer.Minus: sign = -1 p.advance() case lexer.Plus: p.advance() } if p.cur.Kind != lexer.Number { return ast.TimeExpr{}, p.errorf("expected a quoted absolute timestamp or a relative offset like -1h, got %s", p.cur) } n, err := strconv.Atoi(p.cur.Value) if err != nil { return ast.TimeExpr{}, p.errorf("invalid number %q in time expression", p.cur.Value) } p.advance() if p.cur.Kind != lexer.Ident || !isValidTimeUnit(p.cur.Value) { return ast.TimeExpr{}, p.errorf("expected a time unit (s/m/h/d/w) after %d, got %s", n, p.cur) } unit := p.cur.Value p.advance() return ast.TimeExpr{IsRelative: true, RelativeSign: sign, RelativeN: n, RelativeUnit: unit}, nil } func isValidTimeUnit(u string) bool { switch u { case "s", "m", "h", "d", "w": return true default: return false } } func (p *parser) parseExplicitFreeText() (ast.Term, error) { p.advance() // "message" if err := p.expect(lexer.Colon); err != nil { return nil, err } if p.cur.Kind != lexer.String { return nil, p.errorf("expected a quoted string after message:, got %s", p.cur) } v := p.cur.Value p.advance() return ast.FreeText{Query: v}, nil } func (p *parser) parsePipeStage() (ast.PipeStage, error) { if p.cur.Kind != lexer.Ident { return nil, p.errorf("expected a pipe stage (where/stats/sort/fields/head/tail), got %s", p.cur) } switch p.cur.Value { case "where": p.advance() expr, err := p.parseBoolExpr() if err != nil { return nil, err } return ast.WhereStage{Expr: expr}, nil case "stats": return p.parseStatsStage() case "sort": return p.parseSortStage() case "fields": return p.parseFieldsStage() case "head": return p.parseHeadTailStage(false) case "tail": return p.parseHeadTailStage(true) default: return nil, p.errorf("unknown pipe stage %q (expected where/stats/sort/fields/head/tail)", p.cur.Value) } } func (p *parser) parseStatsStage() (ast.PipeStage, error) { p.advance() // "stats" var stage ast.StatsStage agg, err := p.parseAggCall() if err != nil { return nil, err } stage.Aggs = append(stage.Aggs, agg) for p.cur.Kind == lexer.Comma { p.advance() agg, err := p.parseAggCall() if err != nil { return nil, err } stage.Aggs = append(stage.Aggs, agg) } if p.cur.Kind == lexer.Ident && p.cur.Value == "by" { p.advance() field, err := p.parseFieldIdent() if err != nil { return nil, err } stage.By = append(stage.By, field) for p.cur.Kind == lexer.Comma { p.advance() field, err := p.parseFieldIdent() if err != nil { return nil, err } stage.By = append(stage.By, field) } } return stage, nil } func (p *parser) parseAggCall() (ast.AggCall, error) { if p.cur.Kind != lexer.Ident { return ast.AggCall{}, p.errorf("expected an aggregation function (count/sum/avg/min/max), got %s", p.cur) } fn := p.cur.Value if !isValidAggFunc(fn) { return ast.AggCall{}, p.errorf("unknown aggregation function %q (want count/sum/avg/min/max)", fn) } p.advance() // Parens are optional when there's no field: `count`, `count()`, and // `count(*)` are all equivalent. `sum(field)` etc. still need them, // since that's the only way to name the field. var field string if p.cur.Kind == lexer.LParen { p.advance() switch p.cur.Kind { case lexer.Ident: field = p.cur.Value p.advance() case lexer.Star: p.advance() // count(*) is the same as count() } if err := p.expect(lexer.RParen); err != nil { return ast.AggCall{}, err } } var alias string if p.cur.Kind == lexer.Ident && p.cur.Value == "as" { p.advance() if p.cur.Kind != lexer.Ident { return ast.AggCall{}, p.errorf("expected an alias after 'as', got %s", p.cur) } alias = p.cur.Value p.advance() } return ast.AggCall{Func: fn, Field: field, Alias: alias}, nil } func isValidAggFunc(f string) bool { switch f { case "count", "sum", "avg", "min", "max": return true default: return false } } func (p *parser) parseSortStage() (ast.PipeStage, error) { p.advance() // "sort" var stage ast.SortStage field, err := p.parseSortField() if err != nil { return nil, err } stage.Fields = append(stage.Fields, field) for p.cur.Kind == lexer.Comma { p.advance() field, err := p.parseSortField() if err != nil { return nil, err } stage.Fields = append(stage.Fields, field) } return stage, nil } func (p *parser) parseSortField() (ast.SortField, error) { desc := true // no explicit sign defaults to descending, same as an explicit "-" switch p.cur.Kind { case lexer.Minus: p.advance() case lexer.Plus: desc = false p.advance() } if p.cur.Kind != lexer.Ident { return ast.SortField{}, p.errorf("expected a field name in sort, got %s", p.cur) } field := p.cur.Value p.advance() return ast.SortField{Field: field, Desc: desc}, nil } func (p *parser) parseFieldsStage() (ast.PipeStage, error) { p.advance() // "fields" var stage ast.FieldsStage field, err := p.parseFieldIdent() if err != nil { return nil, err } stage.Fields = append(stage.Fields, field) for p.cur.Kind == lexer.Comma { p.advance() field, err := p.parseFieldIdent() if err != nil { return nil, err } stage.Fields = append(stage.Fields, field) } return stage, nil } func (p *parser) parseFieldIdent() (string, error) { if p.cur.Kind != lexer.Ident { return "", p.errorf("expected a field name, got %s", p.cur) } v := p.cur.Value p.advance() return v, nil } func (p *parser) parseHeadTailStage(tail bool) (ast.PipeStage, error) { p.advance() // "head" / "tail" if p.cur.Kind == lexer.Number { n, err := strconv.Atoi(p.cur.Value) if err != nil { return nil, p.errorf("invalid number %q", p.cur.Value) } p.advance() if tail { return ast.TailStage{N: n, HasN: true}, nil } return ast.HeadStage{N: n, HasN: true}, nil } if tail { return ast.TailStage{}, nil } return ast.HeadStage{}, nil } func (p *parser) expect(k lexer.Kind) error { if p.cur.Kind != k { return p.errorf("expected %s, got %s", k, p.cur) } p.advance() return nil } func (p *parser) errorf(format string, args ...any) error { return fmt.Errorf("query syntax error at position %d: %s", p.cur.Pos, fmt.Sprintf(format, args...)) }