Phase 2: unified query language spanning ClickHouse and Tantivy
Replaces the separate SQL-only /query and text-only /search endpoints with one pipe-syntax query language (plus raw SQL escape hatch) that compiles to a single IR and execution plan across both backends, so a query like `message:"connection refused" | stats count by host` runs as one request instead of two disjoint tools. - api/internal/querylang: lexer -> ast -> parser -> ir -> planner -> executor, each layer independently tested. - Execution generalizes Phase 1's proven Tantivy-prefilter pattern into a 4-way routing table (pure ClickHouse / text-only / text + aggregation / raw SQL passthrough). - Unified web query page and `sentryctl query`, both hitting the same POST /query endpoint. - Benchmarked against a real 1,022,000-row dataset (hack/benchmark-fixture); caught and fixed a real bug where the Tantivy prefilter cap (10,000) produced an IN-clause exceeding ClickHouse's default max_query_size -- lowered to 5,000, documented in docs/query-language-design.md and docs/phase-2-runbook.md. - docs/query-language-reference.md: customer-facing syntax reference.
This commit is contained in:
@@ -0,0 +1,18 @@
|
||||
module github.com/sentry/sentry/hack/benchmark-fixture
|
||||
|
||||
go 1.25.0
|
||||
|
||||
replace github.com/sentry/sentry/proto => ../../proto
|
||||
|
||||
require (
|
||||
github.com/sentry/sentry/proto v0.0.0-00010101000000-000000000000
|
||||
google.golang.org/grpc v1.83.0
|
||||
)
|
||||
|
||||
require (
|
||||
golang.org/x/net v0.55.0 // indirect
|
||||
golang.org/x/sys v0.45.0 // indirect
|
||||
golang.org/x/text v0.37.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect
|
||||
google.golang.org/protobuf v1.36.12 // indirect
|
||||
)
|
||||
@@ -0,0 +1,38 @@
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
|
||||
go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
|
||||
go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
|
||||
go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
|
||||
go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
|
||||
go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
|
||||
go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
|
||||
go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
|
||||
golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
|
||||
golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
|
||||
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
|
||||
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
|
||||
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
|
||||
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ=
|
||||
google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ=
|
||||
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
|
||||
google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
@@ -0,0 +1,172 @@
|
||||
// Command benchmark-fixture pushes a large, realistically varied
|
||||
// synthetic dataset directly to ingest's gRPC endpoint, batched, so
|
||||
// Phase 2's "modest dataset" query-latency benchmark
|
||||
// (/docs/phase-2-runbook.md) has real data to measure against instead of
|
||||
// an asserted number. Distinct from /hack/windows-fixture: that one
|
||||
// sends a handful of realistic Windows events to test pipeline
|
||||
// *correctness*; this one sends a lot of Linux-shaped events to test
|
||||
// query *performance* at volume.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"flag"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"os"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials"
|
||||
|
||||
logsv1 "github.com/sentry/sentry/proto/sentry/logs/v1"
|
||||
)
|
||||
|
||||
var (
|
||||
services = []string{"api", "web", "worker", "db", "auth"}
|
||||
hosts = []string{"host-01", "host-02", "host-03", "host-04", "host-05", "host-06", "host-07", "host-08"}
|
||||
severites = []logsv1.Severity{
|
||||
logsv1.Severity_SEVERITY_DEBUG,
|
||||
logsv1.Severity_SEVERITY_INFO,
|
||||
logsv1.Severity_SEVERITY_INFO,
|
||||
logsv1.Severity_SEVERITY_INFO,
|
||||
logsv1.Severity_SEVERITY_WARN,
|
||||
logsv1.Severity_SEVERITY_ERROR,
|
||||
}
|
||||
// A mix of messages, some containing terms worth full-text
|
||||
// searching for (connection refused, timeout) so the benchmark's
|
||||
// text-search-plus-aggregation case has real matches to find, not
|
||||
// just structured rows.
|
||||
messages = []string{
|
||||
"request completed successfully",
|
||||
"connection refused by upstream",
|
||||
"request timeout after 30s",
|
||||
"cache miss, falling back to database",
|
||||
"connection refused: too many open connections",
|
||||
"user authentication succeeded",
|
||||
"slow query detected: timeout approaching",
|
||||
"health check passed",
|
||||
"retrying after connection refused error",
|
||||
"scheduled job completed",
|
||||
}
|
||||
)
|
||||
|
||||
func main() {
|
||||
addr := flag.String("addr", "localhost:4317", "ingest gRPC address")
|
||||
caFile := flag.String("ca", "../dev-certs/out/ca.pem", "CA cert path")
|
||||
certFile := flag.String("cert", "../dev-certs/out/client.pem", "client cert path")
|
||||
keyFile := flag.String("key", "../dev-certs/out/client-key.pem", "client key path")
|
||||
count := flag.Int("count", 1_000_000, "total number of records to generate")
|
||||
batchSize := flag.Int("batch-size", 1000, "records per PushBatch call")
|
||||
concurrency := flag.Int("concurrency", 16, "concurrent PushBatch calls in flight")
|
||||
flag.Parse()
|
||||
|
||||
tlsConf, err := loadTLSConfig(*caFile, *certFile, *keyFile)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "loading TLS config:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// One shared connection: gRPC multiplexes concurrent RPCs over HTTP/2
|
||||
// streams on a single connection, so concurrency here comes from
|
||||
// concurrent PushBatch calls, not from opening more connections.
|
||||
conn, err := grpc.NewClient(*addr, grpc.WithTransportCredentials(credentials.NewTLS(tlsConf)))
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "dialing ingest:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
client := logsv1.NewLogIngestClient(conn)
|
||||
|
||||
numBatches := (*count + *batchSize - 1) / *batchSize
|
||||
batchIndexes := make(chan int, numBatches)
|
||||
for i := 0; i < numBatches; i++ {
|
||||
batchIndexes <- i
|
||||
}
|
||||
close(batchIndexes)
|
||||
|
||||
var sent atomic.Int64
|
||||
var wg sync.WaitGroup
|
||||
start := time.Now()
|
||||
|
||||
for w := 0; w < *concurrency; w++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for batchIdx := range batchIndexes {
|
||||
offset := batchIdx * *batchSize
|
||||
n := *batchSize
|
||||
if remaining := *count - offset; remaining < n {
|
||||
n = remaining
|
||||
}
|
||||
records := make([]*logsv1.LogRecord, n)
|
||||
for i := range records {
|
||||
records[i] = randomRecord()
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
resp, err := client.PushBatch(ctx, &logsv1.PushBatchRequest{
|
||||
BatchId: fmt.Sprintf("benchmark-%d", batchIdx),
|
||||
Records: records,
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "PushBatch %d failed: %v\n", batchIdx, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
total := sent.Add(int64(resp.GetAccepted()))
|
||||
if total%int64(*batchSize*50) < int64(*batchSize) {
|
||||
elapsed := time.Since(start)
|
||||
rate := float64(total) / elapsed.Seconds()
|
||||
fmt.Printf("sent %d/%d (%.0f records/sec)\n", total, *count, rate)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
elapsed := time.Since(start)
|
||||
total := sent.Load()
|
||||
fmt.Printf("done: %d records in %s (%.0f records/sec)\n", total, elapsed, float64(total)/elapsed.Seconds())
|
||||
}
|
||||
|
||||
func randomRecord() *logsv1.LogRecord {
|
||||
return &logsv1.LogRecord{
|
||||
TimestampUnixNano: time.Now().UnixNano(),
|
||||
Host: hosts[rand.Intn(len(hosts))],
|
||||
Service: services[rand.Intn(len(services))],
|
||||
Severity: severites[rand.Intn(len(severites))],
|
||||
Message: messages[rand.Intn(len(messages))],
|
||||
Attributes: map[string]string{
|
||||
"status": fmt.Sprintf("%d", []int{200, 200, 200, 301, 404, 500, 503}[rand.Intn(7)]),
|
||||
"latency_ms": fmt.Sprintf("%d", rand.Intn(2000)),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func loadTLSConfig(caFile, certFile, keyFile string) (*tls.Config, error) {
|
||||
caPEM, err := os.ReadFile(caFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading CA cert %s: %w", caFile, err)
|
||||
}
|
||||
caPool := x509.NewCertPool()
|
||||
if !caPool.AppendCertsFromPEM(caPEM) {
|
||||
return nil, fmt.Errorf("no valid certificates found in %s", caFile)
|
||||
}
|
||||
|
||||
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("loading client cert/key: %w", err)
|
||||
}
|
||||
|
||||
return &tls.Config{
|
||||
RootCAs: caPool,
|
||||
Certificates: []tls.Certificate{cert},
|
||||
}, nil
|
||||
}
|
||||
Reference in New Issue
Block a user