Phase 2: unified query language spanning ClickHouse and Tantivy

Replaces the separate SQL-only /query and text-only /search endpoints with one pipe-syntax query language (plus raw SQL escape hatch) that compiles to a single IR and execution plan across both backends, so a query like `message:"connection refused" | stats count by host` runs as one request instead of two disjoint tools. - api/internal/querylang: lexer -> ast -> parser -> ir -> planner -> executor, each layer independently tested. - Execution generalizes Phase 1's proven Tantivy-prefilter pattern into a 4-way routing table (pure ClickHouse / text-only / text + aggregation / raw SQL passthrough). - Unified web query page and `sentryctl query`, both hitting the same POST /query endpoint. - Benchmarked against a real 1,022,000-row dataset (hack/benchmark-fixture); caught and fixed a real bug where the Tantivy prefilter cap (10,000) produced an IN-clause exceeding ClickHouse's default max_query_size -- lowered to 5,000, documented in docs/query-language-design.md and docs/phase-2-runbook.md. - docs/query-language-reference.md: customer-facing syntax reference.
2026-08-13 12:21:42 -07:00
parent cd8aa290ca
commit fb5049a747
36 changed files with 4119 additions and 613 deletions
@@ -1,6 +1,6 @@
 # sentryctl

-Sentry's control CLI. Phase 0: a single command.
+Sentry's control CLI.

 ```sh
 sentryctl ping                              # checks http://localhost:8080/healthz
@@ -11,9 +11,24 @@ SENTRYCTL_API_URL=http://api.internal:8080 sentryctl ping
 Exits 0 and prints `ok` if `/api`'s `/healthz` responds 200; exits 1 with an
 error on `stderr` otherwise.

-No CLI framework (cobra/urfave-cli/etc.) — a single command doesn't need
-one, and stdlib `os.Args` handling is boring enough not to need a
-dependency. Revisit once there's a real command tree to justify one.
+```sh
+sentryctl query 'service=api | where status>=500 | stats count by host'
+sentryctl query 'SELECT * FROM logs LIMIT 10' --language sql
+sentryctl query 'message:"connection refused"' --json
+```
+
+Quote the query in your shell — pipe syntax uses `|`, which your shell
+interprets as an actual pipe if you don't. Hits the exact same `POST
+/query` endpoint the web UI does (`internal/querylang` in `/api` does the
+compiling; there's no separate query logic here to drift out of sync —
+see `/docs/query-language-reference.md`). `--language` overrides
+auto-detection, same optional override the HTTP API itself exposes.
+Prints a table by default (stdlib `text/tabwriter`, no new dependency);
+`--json` prints the raw `{columns, rows}` response instead.
+
+No CLI framework (cobra/urfave-cli/etc.) — two commands don't need one,
+and stdlib `os.Args` handling is boring enough not to need a dependency.
+Revisit once there's a real command tree to justify one.

 ## Building & testing

@@ -1,13 +1,18 @@
-// Command sentryctl is Sentry's control CLI. Phase 0: a single "ping"
-// command that checks the api service is reachable. More commands land as
-// the control plane grows real operations to expose.
+// Command sentryctl is Sentry's control CLI: "ping" (Phase 0) and
+// "query" (Phase 2), which accepts either query syntax and hits the same
+// POST /query endpoint the web UI does -- no separate query logic here,
+// per the Phase 2 task list's explicit instruction.
 package main

 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
 	"os"
+	"strings"
+	"text/tabwriter"
 	"time"
 )

@@ -26,6 +31,8 @@ func run(args []string, stdout, stderr io.Writer) int {
 	switch args[0] {
 	case "ping":
 		return cmdPing(args[1:], stdout, stderr)
+	case "query":
+		return cmdQuery(args[1:], stdout, stderr)
 	case "-h", "--help", "help":
 		usage(stdout)
 		return 0
@@ -37,15 +44,28 @@ func run(args []string, stdout, stderr io.Writer) int {
 }

 func usage(w io.Writer) {
-	fmt.Fprintln(w, `sentryctl: Sentry control CLI (Phase 0: ping only)
+	fmt.Fprintln(w, `sentryctl: Sentry control CLI

 Usage:
  sentryctl ping [--api <url>]
+  sentryctl query "<query>" [--api <url>] [--language sql|spl] [--json]

 Commands:
-  ping   Checks that the api service is reachable via GET /healthz.
+  ping    Checks that the api service is reachable via GET /healthz.
+  query   Runs a query (pipe syntax or SQL) against POST /query and
+          prints the result as a table, or as JSON with --json. Quote
+          the query in your shell -- pipe syntax uses "|", which your
+          shell will otherwise interpret itself.

--api defaults to $SENTRYCTL_API_URL, or `+defaultAPIURL+` if unset.`)
+--api defaults to $SENTRYCTL_API_URL, or `+defaultAPIURL+` if unset.
+--language overrides auto-detection; omit it for the common case.`)
+}
+
+func resolveAPIURL(env func(string) string) string {
+	if v := env("SENTRYCTL_API_URL"); v != "" {
+		return v
+	}
+	return defaultAPIURL
 }

 // parsePingArgs resolves the api base URL for ping: --api flag wins, then
@@ -53,10 +73,7 @@ Commands:
 // as a function) and separate from the HTTP call so it's unit-testable
 // without a real environment or server.
 func parsePingArgs(args []string, env func(string) string) string {
-	apiURL := env("SENTRYCTL_API_URL")
-	if apiURL == "" {
-		apiURL = defaultAPIURL
-	}
+	apiURL := resolveAPIURL(env)
 	for i := 0; i < len(args); i++ {
 		if args[i] == "--api" && i+1 < len(args) {
 			apiURL = args[i+1]
@@ -85,3 +102,136 @@ func cmdPing(args []string, stdout, stderr io.Writer) int {
 	fmt.Fprintln(stdout, "ok")
 	return 0
 }
+
+type queryArgs struct {
+	apiURL   string
+	jsonOut  bool
+	language string
+	query    string
+}
+
+// parseQueryArgs is pure (env passed in, no I/O), same testability
+// reasoning as parsePingArgs. Non-flag arguments are joined with spaces
+// to form the query, so `sentryctl query service=api status=500` (no
+// quotes, no shell-special characters) works without requiring users to
+// quote every query -- though anything using "|" still needs shell
+// quoting regardless, since that's a real shell pipe character otherwise.
+func parseQueryArgs(args []string, env func(string) string) queryArgs {
+	qa := queryArgs{apiURL: resolveAPIURL(env)}
+	var rest []string
+	for i := 0; i < len(args); i++ {
+		switch args[i] {
+		case "--api":
+			if i+1 < len(args) {
+				qa.apiURL = args[i+1]
+				i++
+			}
+		case "--json":
+			qa.jsonOut = true
+		case "--language":
+			if i+1 < len(args) {
+				qa.language = args[i+1]
+				i++
+			}
+		default:
+			rest = append(rest, args[i])
+		}
+	}
+	qa.query = strings.Join(rest, " ")
+	return qa
+}
+
+type queryRequestBody struct {
+	Query    string `json:"query"`
+	Language string `json:"language"`
+}
+
+type queryResponseBody struct {
+	Columns []string `json:"columns"`
+	Rows    [][]any  `json:"rows"`
+}
+
+type errorResponseBody struct {
+	Error string `json:"error"`
+}
+
+func cmdQuery(args []string, stdout, stderr io.Writer) int {
+	qa := parseQueryArgs(args, os.Getenv)
+	if strings.TrimSpace(qa.query) == "" {
+		fmt.Fprintln(stderr, "sentryctl query: missing query string")
+		return 1
+	}
+
+	reqBody, err := json.Marshal(queryRequestBody{Query: qa.query, Language: qa.language})
+	if err != nil {
+		fmt.Fprintf(stderr, "encoding request: %v\n", err)
+		return 1
+	}
+
+	client := &http.Client{Timeout: 30 * time.Second}
+	resp, err := client.Post(qa.apiURL+"/query", "application/json", bytes.NewReader(reqBody))
+	if err != nil {
+		fmt.Fprintf(stderr, "query failed: %v\n", err)
+		return 1
+	}
+	defer resp.Body.Close()
+
+	respBody, err := io.ReadAll(resp.Body)
+	if err != nil {
+		fmt.Fprintf(stderr, "reading response: %v\n", err)
+		return 1
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		var errResp errorResponseBody
+		if json.Unmarshal(respBody, &errResp) == nil && errResp.Error != "" {
+			fmt.Fprintf(stderr, "query failed: %s\n", errResp.Error)
+		} else {
+			fmt.Fprintf(stderr, "query failed: api returned status %d\n", resp.StatusCode)
+		}
+		return 1
+	}
+
+	if qa.jsonOut {
+		_, _ = stdout.Write(respBody)
+		fmt.Fprintln(stdout)
+		return 0
+	}
+
+	var result queryResponseBody
+	if err := json.Unmarshal(respBody, &result); err != nil {
+		fmt.Fprintf(stderr, "decoding response: %v\n", err)
+		return 1
+	}
+	printTable(stdout, result.Columns, result.Rows)
+	return 0
+}
+
+func printTable(w io.Writer, columns []string, rows [][]any) {
+	tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0)
+	fmt.Fprintln(tw, strings.Join(columns, "\t"))
+	for _, row := range rows {
+		cells := make([]string, len(row))
+		for i, v := range row {
+			cells[i] = formatCell(v)
+		}
+		fmt.Fprintln(tw, strings.Join(cells, "\t"))
+	}
+	_ = tw.Flush()
+	fmt.Fprintf(w, "(%d row(s))\n", len(rows))
+}
+
+func formatCell(v any) string {
+	switch t := v.(type) {
+	case nil:
+		return ""
+	case map[string]any, []any:
+		b, err := json.Marshal(t)
+		if err != nil {
+			return fmt.Sprintf("%v", t)
+		}
+		return string(b)
+	default:
+		return fmt.Sprintf("%v", t)
+	}
+}
@@ -2,6 +2,7 @@ package main

 import (
 	"bytes"
+	"encoding/json"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -121,3 +122,129 @@ func TestRunHelp(t *testing.T) {
 		t.Fatalf("stdout should contain usage text, got %q", stdout.String())
 	}
 }
+
+func TestParseQueryArgsJoinsNonFlagArgsAsQuery(t *testing.T) {
+	env := func(string) string { return "" }
+	qa := parseQueryArgs([]string{"service=api", "status=500"}, env)
+	if qa.query != "service=api status=500" {
+		t.Errorf("query = %q", qa.query)
+	}
+	if qa.apiURL != defaultAPIURL {
+		t.Errorf("apiURL = %q, want default", qa.apiURL)
+	}
+	if qa.jsonOut {
+		t.Error("jsonOut should default to false")
+	}
+}
+
+func TestParseQueryArgsFlags(t *testing.T) {
+	env := func(string) string { return "" }
+	qa := parseQueryArgs([]string{"--api", "http://h:1", "--language", "sql", "--json", "SELECT", "1"}, env)
+	if qa.apiURL != "http://h:1" {
+		t.Errorf("apiURL = %q", qa.apiURL)
+	}
+	if qa.language != "sql" {
+		t.Errorf("language = %q", qa.language)
+	}
+	if !qa.jsonOut {
+		t.Error("expected jsonOut = true")
+	}
+	if qa.query != "SELECT 1" {
+		t.Errorf("query = %q", qa.query)
+	}
+}
+
+func TestCmdQueryMissingQueryErrors(t *testing.T) {
+	var stdout, stderr bytes.Buffer
+	code := cmdQuery(nil, &stdout, &stderr)
+	if code != 1 {
+		t.Fatalf("exit code = %d, want 1", code)
+	}
+	if !strings.Contains(stderr.String(), "missing query") {
+		t.Fatalf("stderr = %q", stderr.String())
+	}
+}
+
+func TestCmdQueryTableOutput(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/query" {
+			t.Errorf("unexpected path %q", r.URL.Path)
+		}
+		var body queryRequestBody
+		if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+			t.Fatalf("decoding request: %v", err)
+		}
+		if body.Query != "service=api" {
+			t.Errorf("query = %q", body.Query)
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(queryResponseBody{
+			Columns: []string{"host", "count"},
+			Rows:    [][]any{{"h1", float64(3)}},
+		})
+	}))
+	defer srv.Close()
+
+	var stdout, stderr bytes.Buffer
+	code := cmdQuery([]string{"--api", srv.URL, "service=api"}, &stdout, &stderr)
+
+	if code != 0 {
+		t.Fatalf("exit code = %d, want 0; stderr=%s", code, stderr.String())
+	}
+	out := stdout.String()
+	if !strings.Contains(out, "host") || !strings.Contains(out, "h1") || !strings.Contains(out, "(1 row(s))") {
+		t.Fatalf("unexpected table output: %q", out)
+	}
+}
+
+func TestCmdQueryJSONOutput(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(queryResponseBody{Columns: []string{"c"}, Rows: [][]any{{"v"}}})
+	}))
+	defer srv.Close()
+
+	var stdout, stderr bytes.Buffer
+	code := cmdQuery([]string{"--api", srv.URL, "--json", "service=api"}, &stdout, &stderr)
+
+	if code != 0 {
+		t.Fatalf("exit code = %d, want 0; stderr=%s", code, stderr.String())
+	}
+	var got queryResponseBody
+	if err := json.Unmarshal(stdout.Bytes(), &got); err != nil {
+		t.Fatalf("stdout is not valid JSON: %v; got %q", err, stdout.String())
+	}
+	if len(got.Columns) != 1 || got.Columns[0] != "c" {
+		t.Fatalf("unexpected JSON output: %+v", got)
+	}
+}
+
+func TestCmdQueryServerErrorPrintsMessage(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusBadRequest)
+		_ = json.NewEncoder(w).Encode(errorResponseBody{Error: "only SELECT queries are allowed"})
+	}))
+	defer srv.Close()
+
+	var stdout, stderr bytes.Buffer
+	code := cmdQuery([]string{"--api", srv.URL, "DELETE FROM logs", "--language", "sql"}, &stdout, &stderr)
+
+	if code != 1 {
+		t.Fatalf("exit code = %d, want 1", code)
+	}
+	if !strings.Contains(stderr.String(), "only SELECT queries are allowed") {
+		t.Fatalf("stderr = %q, want it to include the server's error message", stderr.String())
+	}
+}
+
+func TestFormatCellHandlesNilMapAndSlice(t *testing.T) {
+	if got := formatCell(nil); got != "" {
+		t.Errorf("formatCell(nil) = %q, want empty", got)
+	}
+	if got := formatCell(map[string]any{"a": "b"}); got != `{"a":"b"}` {
+		t.Errorf("formatCell(map) = %q", got)
+	}
+	if got := formatCell(42.0); got != "42" {
+		t.Errorf("formatCell(42.0) = %q", got)
+	}
+}