From cd8aa290ca056a36069367e6e58e53f1c412e2ea Mon Sep 17 00:00:00 2001 From: John Coffey Date: Thu, 13 Aug 2026 11:27:35 -0700 Subject: [PATCH] Phase 1: Windows log collection + full-text search Extends the agent, ingest, storage, api, and web with Windows Event Log/ETW sourcing and Tantivy-backed free-text search, per the approved Phase 1 plan. - CLAUDE.md: materialized on disk (never existed as a file before) with a new Phase 1 "done looks like" section. - agent: Windows Event Log (EvtSubscribe) and ETW sources, Windows service wrapper (install/uninstall/run-service), both feature- and target_os-gated so Linux builds/tests/clippy stay unaffected. Also fixed two pre-existing Phase 0 clippy gaps (dead-code on default-features-only builds, a type-inference edge case) found while testing every feature combination properly for the first time. UNVERIFIED on real Windows -- no Windows toolchain existed anywhere in the build environment; flagged prominently in three places. - proto/ingest: new record_id field, assigned once server-side in ingest's gRPC front end so ClickHouse and Tantivy agree on the same ID for the same record. - storage: record_id column + bloom filter index, verified against a live ClickHouse. - search: new service, Tantivy index, rskafka consumer as an independent second consumer group on the same Redpanda topic ingest already reads. - api/web: new /search endpoint and page, sharing the query page's result-table shape and component. - hack/windows-fixture: sends realistic Windows-shaped data straight to ingest, so the pipeline's handling of it is verifiable without a Windows host. Verified end-to-end on the live docker-compose stack: the same record_id comes back from both /query and /search for the same log line, including for windows-fixture's synthetic Windows Event Log data. Real bugs found and fixed along the way: api/Dockerfile missing proto/ in its build context, search's logs being completely silent (RUST_LOG gap), and search/target/ missing from .gitignore/.dockerignore. --- .dockerignore | 5 +- .gitignore | 2 + CLAUDE.md | 76 + agent/Cargo.lock | 93 + agent/README.md | 162 +- agent/sentry-agent/Cargo.toml | 21 +- agent/sentry-agent/config/agent.example.toml | 26 +- agent/sentry-agent/src/batch.rs | 1 + agent/sentry-agent/src/config.rs | 73 +- agent/sentry-agent/src/main.rs | 116 +- agent/sentry-agent/src/service.rs | 170 ++ agent/sentry-agent/src/source/etw.rs | 248 ++ agent/sentry-agent/src/source/file_tail.rs | 1 + agent/sentry-agent/src/source/journald.rs | 1 + agent/sentry-agent/src/source/mod.rs | 24 +- .../src/source/windows_eventlog.rs | 283 ++ api/Dockerfile | 5 +- api/README.md | 60 +- api/cmd/api/main.go | 18 +- api/go.mod | 14 +- api/go.sum | 26 + api/internal/config/config.go | 4 + api/internal/config/config_test.go | 3 + api/internal/queryapi/handler.go | 31 +- api/internal/queryapi/handler_test.go | 18 +- api/internal/queryapi/search.go | 96 + api/internal/queryapi/search_test.go | 125 + api/internal/searchclient/client.go | 44 + docker-compose.yml | 39 +- docs/phase-0-runbook.md | 28 +- docs/phase-1-runbook.md | 243 ++ hack/README.md | 5 + hack/windows-fixture/README.md | 52 + hack/windows-fixture/go.mod | 18 + hack/windows-fixture/go.sum | 38 + hack/windows-fixture/main.go | 130 + ingest/README.md | 18 +- ingest/go.mod | 2 +- ingest/internal/clickhousewriter/writer.go | 4 +- ingest/internal/grpcserver/server.go | 18 +- ingest/internal/grpcserver/server_test.go | 126 + ingest/internal/normalize/normalize.go | 10 + ingest/internal/normalize/normalize_test.go | 15 + proto/sentry/logs/v1/logs.pb.go | 26 +- proto/sentry/logs/v1/logs.proto | 13 +- proto/sentry/search/v1/search.pb.go | 192 ++ proto/sentry/search/v1/search.proto | 33 + proto/sentry/search/v1/search_grpc.pb.go | 133 + search/Cargo.lock | 2354 +++++++++++++++++ search/Cargo.toml | 33 + search/Dockerfile | 20 + search/README.md | 100 + search/build.rs | 14 + search/src/config.rs | 44 + search/src/consumer.rs | 134 + search/src/grpc.rs | 48 + search/src/index.rs | 192 ++ search/src/main.rs | 67 + search/src/offsets.rs | 93 + storage/README.md | 23 +- storage/migrations/0002_add_record_id.sql | 3 + web/src/lib/ResultsTable.svelte | 59 + web/src/routes/+layout.svelte | 29 + web/src/routes/+page.svelte | 53 +- web/src/routes/search/+page.svelte | 96 + web/src/routes/search/+page.ts | 4 + 66 files changed, 6084 insertions(+), 171 deletions(-) create mode 100644 CLAUDE.md create mode 100644 agent/sentry-agent/src/service.rs create mode 100644 agent/sentry-agent/src/source/etw.rs create mode 100644 agent/sentry-agent/src/source/windows_eventlog.rs create mode 100644 api/internal/queryapi/search.go create mode 100644 api/internal/queryapi/search_test.go create mode 100644 api/internal/searchclient/client.go create mode 100644 docs/phase-1-runbook.md create mode 100644 hack/windows-fixture/README.md create mode 100644 hack/windows-fixture/go.mod create mode 100644 hack/windows-fixture/go.sum create mode 100644 hack/windows-fixture/main.go create mode 100644 ingest/internal/grpcserver/server_test.go create mode 100644 proto/sentry/search/v1/search.pb.go create mode 100644 proto/sentry/search/v1/search.proto create mode 100644 proto/sentry/search/v1/search_grpc.pb.go create mode 100644 search/Cargo.lock create mode 100644 search/Cargo.toml create mode 100644 search/Dockerfile create mode 100644 search/README.md create mode 100644 search/build.rs create mode 100644 search/src/config.rs create mode 100644 search/src/consumer.rs create mode 100644 search/src/grpc.rs create mode 100644 search/src/index.rs create mode 100644 search/src/main.rs create mode 100644 search/src/offsets.rs create mode 100644 storage/migrations/0002_add_record_id.sql create mode 100644 web/src/lib/ResultsTable.svelte create mode 100644 web/src/routes/search/+page.svelte create mode 100644 web/src/routes/search/+page.ts diff --git a/.dockerignore b/.dockerignore index b6d1cda..d582040 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,8 @@ -# ingest/ and api/ build with context "." (repo root) so their Dockerfiles -# can also COPY proto/. Keep that context lean. +# ingest/, api/, and search/ build with context "." (repo root) so their +# Dockerfiles can also COPY proto/. Keep that context lean. .git/ agent/target/ +search/target/ web/node_modules/ web/build/ web/.svelte-kit/ diff --git a/.gitignore b/.gitignore index ec986ec..b1c737d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,13 @@ # Rust agent/target/ +search/target/ # Go build cache (go build ./... without -o doesn't normally leave # binaries in-tree, but be defensive) /ingest/ingest /api/api /cli/sentryctl +/hack/windows-fixture/windows-fixture # Node / SvelteKit (web/ has its own more detailed .gitignore too) web/node_modules/ diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..3410aab --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,76 @@ +# Project: Sentry — Distributed Log Aggregation & Observability Platform + +## Mission +Build an open-core, Kubernetes-native centralized logging platform that rivals +Splunk on features but wins on cost-per-GB, modern language stack, and honest +multi-tenant RBAC. Full architecture spec is in `/docs/architecture.md` — read +it before touching any component. Do not deviate from the storage/query split +described there without flagging it to me first. + +## Non-negotiable constraints +- Distro-agnostic Linux agent: must run identically on RHEL/Debian/Arch/SUSE + derivatives via a statically-linked musl binary. No glibc runtime deps. +- Windows support via native ETW/Event Log API, not a WSL shim. +- AGPLv3 for core + agents. Enterprise module (SSO/multi-tenancy/compliance) + lives in a separate `enterprise/` directory under a commercial license stub + — keep the boundary clean from day one, don't let AGPL code import from it. +- Schema-on-write with OTel semantic conventions as the default schema, with + schema-on-read fallback for unstructured text. +- Every UI action must correspond to a documented REST/gRPC call. No + UI-only logic. CLI (`sentryctl`) and Terraform provider are first-class, + not afterthoughts. + +## Tech stack (pinned — do not substitute without discussion) +| Component | Language/Tool | +|-------------------|------------------------| +| Edge agent | Rust, musl target | +| Transport | Redpanda (Kafka API) | +| Ingest/parse | Go | +| Analytical store | ClickHouse | +| Full-text index | Tantivy (Rust) | +| Control plane/API | Go, gRPC + REST gateway | +| Frontend | SvelteKit + TypeScript | +| Deployment | Kubernetes Operator (Go, kubebuilder), Helm, docker-compose for local/homelab | + +## Repo conventions +- Monorepo, one top-level dir per component (see structure below). +- Rust: workspace-based, `cargo clippy --all-targets -- -D warnings` must pass. +- Go: standard `go vet` + `golangci-lint`, no globals for shared state. +- Every component ships with: unit tests, a `README.md`, and a Dockerfile + using distroless or scratch base images where feasible. +- Conventional commits. Every PR-sized change should be a logically complete, + independently revertible unit. +- Prefer boring, well-understood dependencies over novel ones. This is + infrastructure software; operators need to trust it. + +## What "done" looks like for Phase 0 (MVP) + +**Status: shipped.** A single log line, generated on a Linux host by the +Rust agent, flows: agent → Redpanda → Go ingest service → ClickHouse, and +is queryable via a minimal SQL endpoint and visible in a bare-bones +SvelteKit table view. Verified end-to-end on real hardware, not just in +CI — see `/docs/phase-0-runbook.md`. No alerting, no multi-tenancy, no +dashboards — that discipline held for the whole phase. + +## What "done" looks like for Phase 1 + +A Windows Event Log entry and a Linux journald entry should both be +queryable via SQL (the ClickHouse path) and via free-text search (the +Tantivy path), from the same UI, within a few seconds of being generated. + +Non-goals for this phase (same "resist scope creep" discipline as Phase 0): +no alerting, no dashboards, no SPL-like query layer, no multi-tenancy, and +no unified query experience — two separate boxes on two separate pages is +correct for Phase 1; unifying them is Phase 2's job. + +ETW and WEF (Windows Event Forwarding) are *designed* in this phase but not +required to be running for "done": ETW ships behind a feature flag most +environments won't enable (it needs elevated privileges), and WEF's +receiver-side is explicitly deferred rather than built now — see +`/docs/phase-1-runbook.md` for both. Only the Event Log source needs to +actually be running end-to-end for this phase to count as done. + +## When in doubt +Ask before: changing the pinned stack, adding a new external dependency +that pulls in a large transitive tree, or making an architectural decision +that isn't already specified in `/docs/architecture.md`. diff --git a/agent/Cargo.lock b/agent/Cargo.lock index b293676..4329c5e 100644 --- a/agent/Cargo.lock +++ b/agent/Cargo.lock @@ -735,6 +735,15 @@ dependencies = [ "prost", ] +[[package]] +name = "quick-xml" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.47" @@ -893,6 +902,7 @@ dependencies = [ "anyhow", "clap", "prost", + "quick-xml", "sentry-parser", "serde", "serde_json", @@ -902,6 +912,8 @@ dependencies = [ "tonic-build", "tracing", "tracing-subscriber", + "windows", + "windows-service", ] [[package]] @@ -1380,12 +1392,93 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + +[[package]] +name = "windows" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" +dependencies = [ + "windows-core", + "windows-targets", +] + +[[package]] +name = "windows-core" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-result", + "windows-strings", + "windows-targets", +] + +[[package]] +name = "windows-implement" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-service" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24d6bcc7f734a4091ecf8d7a64c5f7d7066f45585c1861eba06449909609c8a" +dependencies = [ + "bitflags", + "widestring", + "windows-sys 0.52.0", +] + +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result", + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.52.0" diff --git a/agent/README.md b/agent/README.md index cdc7a5f..0f1d189 100644 --- a/agent/README.md +++ b/agent/README.md @@ -1,15 +1,39 @@ # sentry-agent -Distro-agnostic Linux log collector. Statically linked against musl, no -glibc runtime dependency. Tails journald (default) or a file, batches -lines, and ships them over mTLS gRPC to the ingest service. +Distro-agnostic Linux/Windows log collector. On Linux, statically linked +against musl, no glibc runtime dependency. Tails journald (Linux default), +a file, Windows Event Log, or ETW, batches lines, and ships them over mTLS +gRPC to the ingest service. + +**Windows support status:** the Windows-specific code +(`source/windows_eventlog.rs`, `source/etw.rs`, `service.rs`) was written +against documented Win32/ETW API shapes but has **not been compiled or run +on Windows** — no Windows toolchain was available in the environment this +was built in (confirmed: only the Linux target's std library was +installed, no way to even `cargo check --target x86_64-pc-windows-*`). +Linux builds/tests/clippy are verified clean across every feature +combination; Windows code is a first draft to compile-check and test for +real before trusting it. See `/docs/phase-1-runbook.md`. ## Workspace layout - `sentry-parser` — pure-`std` RFC 5424 syslog parser with raw-passthrough fallback. No I/O, easy to unit test in isolation. -- `sentry-agent` — the binary: config loading, sourcing (journald/file), - batching, mTLS gRPC client. +- `sentry-agent` — the binary: config loading, sourcing (journald/file/ + Windows Event Log/ETW), batching, mTLS gRPC client, Windows service + wrapper. + +## Why one crate for both platforms, not a platform split + +`config.rs`, `batch.rs`, `grpc.rs`, and `main.rs`'s event loop are already +100% cross-platform Rust — nothing in them is Linux- or Windows-specific. +Only the `source/` modules differ per platform, and that boundary already +existed before Windows support was added (it's exactly what made adding +Windows sources a matter of adding two files, not restructuring anything). +Windows-only dependencies (`windows`, `windows-service`, `quick-xml`) live +in a `[target.'cfg(windows)'.dependencies]` section in `Cargo.toml`, so +they're not in the Linux build's dependency graph at all — no crate split +needed to keep the two platforms from stepping on each other. ## Why journalctl, not libsystemd @@ -62,6 +86,32 @@ container without deliberately bind-mounting `/var/log/journal` (or deployment for journald sourcing is as a native binary managed by systemd on the host, not containerized. +### Building for Windows + +```sh +# Cross-compiling FROM Linux, for the build step only: +rustup target add x86_64-pc-windows-gnu +cargo build --release --target x86_64-pc-windows-gnu \ + --no-default-features --features windows-eventlog,etw + +# Natively on Windows (MSVC toolchain): +cargo build --release --target x86_64-pc-windows-msvc \ + --no-default-features --features windows-eventlog,etw +``` + +`--no-default-features` matters: the default feature set is `journald`, +which is Linux-only (the module is `target_os = "linux"`-gated and simply +won't compile in on Windows, but there's no reason to carry the dead +feature flag). Drop `,etw` from `--features` if you only want Event Log — +see the privilege note below for why most environments will want to. + +**Cross-compilation only covers the *build* step.** Running/testing the +Windows sources — actually calling `EvtSubscribe`, starting an ETW +session, registering a Windows service — needs a real or virtualized +Windows host. There is no way around that, and nothing in this repo +pretends otherwise; see `/docs/phase-1-runbook.md` for exactly what's +automatable vs. manual-only. + ## Running No CLI flags are required for the common case: @@ -70,12 +120,14 @@ No CLI flags are required for the common case: ./sentry-agent ``` -This uses `/etc/sentry-agent/agent.toml` if present, otherwise built-in -defaults: journald source (whole journal, no unit filter), service name -`default`, and mTLS material expected at -`/etc/sentry-agent/{ca,client,client-key}.pem`. mTLS is mandatory per the -project's transport requirements, so a from-scratch run with no certs in -place will fail fast with a clear error rather than connecting insecurely. +This uses the platform's conventional config path if present +(`/etc/sentry-agent/agent.toml` on Linux, `C:\ProgramData\SentryAgent\agent.toml` +on Windows), otherwise built-in defaults: journald source on Linux (whole +journal, no unit filter), service name `default`, and mTLS material +expected under the same conventional directory +(`{ca,client,client-key}.pem`). mTLS is mandatory per the project's +transport requirements, so a from-scratch run with no certs in place will +fail fast with a clear error rather than connecting insecurely. See `config/agent.example.toml` for all fields. @@ -83,6 +135,70 @@ See `config/agent.example.toml` for all fields. ./sentry-agent --config /path/to/agent.toml ``` +## Running as a Windows service + +"A native Windows service, not a WSL wrapper" means implementing the Win32 +Service Control Manager protocol, not just running the binary in a +console — that's what `service.rs` (via the `windows-service` crate) +does. From an administrator shell: + +```powershell +sentry-agent.exe install # registers the service, Automatic start, LocalSystem account +sc.exe start SentryAgent +sc.exe stop SentryAgent +sentry-agent.exe uninstall +``` + +`install`/`uninstall`/`run-service` are subcommands only present in +Windows builds (`sentry-agent` with no subcommand is still the normal +foreground/console run, same as on Linux) — `run-service` specifically is +what the SCM itself invokes at service start; don't run it directly. + +**Known limitation:** when running as a service, there's no console +attached, so `tracing_subscriber::fmt()`'s stdout writer has nowhere to +go — logs won't be visible anywhere useful until this is redirected to a +file or a proper Windows Event Log tracing sink is written. Not addressed +in Phase 1; flagging it here rather than shipping it silently broken. + +## ETW: read this before enabling it + +ETW needs elevated privileges to subscribe to most providers — running +the agent under an administrator token or a service account with +`SeSystemProfilePrivilege`/ETW-specific rights. This is a real privilege +escalation, not a footnote: think about whether your environment wants +the log-shipping agent running with that level of access before turning +on the `etw` feature and an `[source] kind = "etw"` config. Event Log +alone (no elevated privileges needed) covers the common case and is what +Phase 1's exit criteria in `/CLAUDE.md` actually requires to be running. + +Providers are configured by **GUID**, not friendly name — ETW's own API +requires it. Look one up with `logman query providers ""`. + +## Windows Event Forwarding (WEF) + +Two different things people mean by "WEF support," worth being explicit +about since they're very different amounts of work: + +1. **What this repo supports today, with zero extra code:** WEF is a + native Windows-to-Windows mechanism (`wecsvc`, the built-in Windows + Event Collector role) — endpoints forward to a Windows Server acting + as collector using Windows' own mechanism, no Sentry code involved in + the forwarding itself. Run this agent *on the collector box*, + subscribed to the `ForwardedEvents` channel instead of the usual three: + ```toml + [source] + kind = "eventlog" + channels = ["ForwardedEvents"] + ``` +2. **What this repo does *not* implement:** a true agentless receiver — + Sentry itself speaking the WS-Management/WinRM event-subscription + protocol so endpoints can forward directly to `ingest` without any + Windows Event Collector role or Sentry agent anywhere. That's a + standalone protocol implementation (SOAP-ish subscription/heartbeat/ + delivery over WinRM), not an agent or ingest-side tweak, and it's out + of scope for Phase 1. If you need this, it's a real project of its + own — say so before assuming it's a small addition. + ## Testing ```sh @@ -91,10 +207,24 @@ cargo test --workspace ## Feature flags -- `journald` (default) — journalctl-based journald source. +- `journald` (default) — journalctl-based journald source. `target_os = + "linux"`-gated: enabling this on a Windows build is a no-op, not a + build failure. - `file-tail` — polling-based file tailer (no inotify dependency; doesn't - follow rename-based log rotation yet). + follow rename-based log rotation yet). Cross-platform, works on Windows + too. +- `windows-eventlog` — Windows Event Log via `EvtSubscribe`. + `target_os = "windows"`-gated the same way; a no-op on Linux. +- `etw` — ETW real-time session. Same gating. See the privilege section + above before enabling. -Both can be enabled together; `[source].kind` in config picks which one -runs. Building without a feature and configuring that source at runtime -fails at startup with a clear error rather than silently doing nothing. +Any combination can be enabled together; `[source].kind` in config picks +which one actually runs. Building without a feature and configuring that +source at runtime fails at startup with a clear error rather than +silently doing nothing. + +Dependencies added for Windows support, worth knowing about: +`windows` (Microsoft's official Win32/ETW bindings), `windows-service` +(Windows Service Control Manager wrapper), `quick-xml` (parses +EvtSubscribe's rendered event XML). All three are `[target.'cfg(windows)'.dependencies]` +— not in the Linux build's dependency graph at all. diff --git a/agent/sentry-agent/Cargo.toml b/agent/sentry-agent/Cargo.toml index ace6482..2076de0 100644 --- a/agent/sentry-agent/Cargo.toml +++ b/agent/sentry-agent/Cargo.toml @@ -3,7 +3,7 @@ name = "sentry-agent" version.workspace = true edition.workspace = true license.workspace = true -description = "Sentry distro-agnostic Linux log collector" +description = "Sentry distro-agnostic Linux/Windows log collector" [[bin]] name = "sentry-agent" @@ -13,6 +13,12 @@ path = "src/main.rs" default = ["journald"] journald = [] file-tail = [] +# Windows-only sources. Feature-enabled AND target_os="windows"-gated at +# the module level (see src/source/mod.rs), so enabling these on a +# non-Windows build is a harmless no-op, not a build failure -- keeps +# `cargo test --workspace --all-features` green on Linux CI. +windows-eventlog = [] +etw = [] [dependencies] sentry-parser = { path = "../sentry-parser" } @@ -30,5 +36,18 @@ anyhow = "1" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } +# Windows-only: not in the dependency graph at all on other targets, so +# they don't affect Linux build times or the musl release binary. +[target.'cfg(windows)'.dependencies] +windows = { version = "0.58", features = [ + "Win32_Foundation", + "Win32_System_EventLog", + "Win32_System_Threading", + "Win32_System_Diagnostics_Etw", + "Win32_Security", +] } +windows-service = "0.7" +quick-xml = "0.36" + [build-dependencies] tonic-build = "0.12" diff --git a/agent/sentry-agent/config/agent.example.toml b/agent/sentry-agent/config/agent.example.toml index d0013db..6331ce5 100644 --- a/agent/sentry-agent/config/agent.example.toml +++ b/agent/sentry-agent/config/agent.example.toml @@ -1,25 +1,39 @@ -# Example sentry-agent config. Copy to /etc/sentry-agent/agent.toml, or -# pass --config /path/to/this/file. +# Example sentry-agent config. Copy to the platform's conventional path +# (/etc/sentry-agent/agent.toml on Linux, C:\ProgramData\SentryAgent\agent.toml +# on Windows), or pass --config /path/to/this/file. # # Every field has a built-in default (see src/config.rs), so this file only # needs to contain what you're overriding. An agent with NO config file at -# all still runs: it defaults to journald, service = "default", and expects -# mTLS material at /etc/sentry-agent/{ca,client,client-key}.pem. +# all still runs: on Linux it defaults to journald, service = "default", +# and expects mTLS material at /etc/sentry-agent/{ca,client,client-key}.pem +# (Windows equivalents under C:\ProgramData\SentryAgent\). [agent] -# host = "explicit-hostname-override" # defaults to /etc/hostname +# host = "explicit-hostname-override" # defaults to /etc/hostname (Linux) or %COMPUTERNAME% (Windows) service = "my-service" [source] kind = "journald" # unit = "nginx.service" # omit to tail the whole journal -# To tail a file instead: +# To tail a file instead (works on both Linux and Windows): # [source] # kind = "file" # path = "/var/log/nginx/access.log" # from_beginning = false +# Windows Event Log (requires the agent to be built with the +# `windows-eventlog` feature — see /agent/README.md): +# [source] +# kind = "eventlog" +# channels = ["Application", "System", "Security"] # this is the default if omitted + +# ETW (requires the `etw` feature, and usually elevated privileges — read +# /agent/README.md's privilege section before enabling this): +# [source] +# kind = "etw" +# providers = ["{22FB2CD6-0E7B-422B-A0C7-2FAD1FD0E716}"] # GUIDs, not friendly names + [batch] max_size = 500 flush_interval_ms = 2000 diff --git a/agent/sentry-agent/src/batch.rs b/agent/sentry-agent/src/batch.rs index 0db36f6..075063c 100644 --- a/agent/sentry-agent/src/batch.rs +++ b/agent/sentry-agent/src/batch.rs @@ -63,6 +63,7 @@ mod tests { severity: 0, message: msg.into(), attributes: Default::default(), + record_id: String::new(), } } diff --git a/agent/sentry-agent/src/config.rs b/agent/sentry-agent/src/config.rs index 9f0e40c..bd291ef 100644 --- a/agent/sentry-agent/src/config.rs +++ b/agent/sentry-agent/src/config.rs @@ -2,7 +2,10 @@ use anyhow::{Context, Result}; use serde::Deserialize; use std::path::{Path, PathBuf}; +#[cfg(not(windows))] const DEFAULT_CONFIG_PATH: &str = "/etc/sentry-agent/agent.toml"; +#[cfg(windows)] +const DEFAULT_CONFIG_PATH: &str = r"C:\ProgramData\SentryAgent\agent.toml"; #[derive(Debug, Clone, Deserialize, Default)] #[serde(default)] @@ -15,11 +18,13 @@ pub struct Config { } impl Config { - /// Loads config from `explicit_path` if given, else from - /// `/etc/sentry-agent/agent.toml` if it exists, else falls back to - /// built-in defaults (journald source, default TLS cert paths). Only an - /// explicitly-passed `--config` path that doesn't exist is an error; - /// the conventional default path is optional. + /// Loads config from `explicit_path` if given, else from the + /// platform's conventional config path if it exists + /// (`/etc/sentry-agent/agent.toml` on Linux, + /// `C:\ProgramData\SentryAgent\agent.toml` on Windows), else falls + /// back to built-in defaults (journald source on Linux, default TLS + /// cert paths). Only an explicitly-passed `--config` path that doesn't + /// exist is an error; the conventional default path is optional. pub fn load(explicit_path: Option<&Path>) -> Result { let path = match explicit_path { Some(p) => Some(p.to_path_buf()), @@ -63,13 +68,55 @@ impl Default for AgentConfig { pub enum SourceConfig { Journald { #[serde(default)] + #[cfg_attr(not(all(feature = "journald", target_os = "linux")), allow(dead_code))] unit: Option, }, + // Dead-code-on-default-build, same reasoning as EventLog/Etw below: + // these fields are only read by the `file-tail`-gated arm in + // spawn_source (main.rs), which doesn't exist in the default build + // (`default = ["journald"]`). Pre-existing gap from Phase 0 — CLAUDE.md + // mandates plain `cargo clippy --all-targets -- -D warnings` (no + // --all-features), which this broke silently since only + // --all-features clippy was ever actually run. File { + #[cfg_attr(not(feature = "file-tail"), allow(dead_code))] path: PathBuf, #[serde(default)] + #[cfg_attr(not(feature = "file-tail"), allow(dead_code))] from_beginning: bool, }, + /// Windows Event Log via EvtSubscribe. Requires the agent to be built + /// with the `windows-eventlog` feature; see /agent/README.md. + /// + /// `channels`/`providers` below are read only by the Windows-only + /// consumers in `spawn_source` (main.rs), which don't exist at all on + /// non-Windows builds — unlike `File`'s fields (dead only when the + /// `file-tail` feature happens to be off), these are dead on *every* + /// non-Windows build regardless of feature flags, since their sole + /// consumer is `target_os = "windows"`-gated. `cfg_attr` here keeps + /// clippy honest: still flags genuine dead code on an actual Windows + /// build, just not on the platform where these fields can never be + /// read no matter what. + EventLog { + #[serde(default = "default_eventlog_channels")] + #[cfg_attr(not(windows), allow(dead_code))] + channels: Vec, + }, + /// ETW (Event Tracing for Windows). Requires the `etw` feature and + /// (usually) elevated privileges — see /agent/README.md before + /// enabling this in any environment that isn't Windows-first. + Etw { + #[cfg_attr(not(windows), allow(dead_code))] + providers: Vec, + }, +} + +fn default_eventlog_channels() -> Vec { + vec![ + "Application".to_string(), + "System".to_string(), + "Security".to_string(), + ] } impl Default for SourceConfig { @@ -119,9 +166,19 @@ pub struct TlsConfig { impl Default for TlsConfig { fn default() -> Self { Self { - ca_cert: PathBuf::from("/etc/sentry-agent/ca.pem"), - client_cert: PathBuf::from("/etc/sentry-agent/client.pem"), - client_key: PathBuf::from("/etc/sentry-agent/client-key.pem"), + ca_cert: default_cert_path("ca.pem"), + client_cert: default_cert_path("client.pem"), + client_key: default_cert_path("client-key.pem"), } } } + +#[cfg(not(windows))] +fn default_cert_path(name: &str) -> PathBuf { + PathBuf::from(format!("/etc/sentry-agent/{name}")) +} + +#[cfg(windows)] +fn default_cert_path(name: &str) -> PathBuf { + PathBuf::from(format!(r"C:\ProgramData\SentryAgent\{name}")) +} diff --git a/agent/sentry-agent/src/main.rs b/agent/sentry-agent/src/main.rs index 2347712..d933e42 100644 --- a/agent/sentry-agent/src/main.rs +++ b/agent/sentry-agent/src/main.rs @@ -3,6 +3,9 @@ mod config; mod grpc; mod source; +#[cfg(windows)] +mod service; + pub mod pb { tonic::include_proto!("sentry.logs.v1"); } @@ -18,23 +21,66 @@ use tokio::sync::mpsc; use tonic::transport::Channel; #[derive(Parser)] -#[command(name = "sentry-agent", about = "Sentry Linux log collector")] +#[command(name = "sentry-agent", about = "Sentry Linux/Windows log collector")] struct Cli { - /// Path to a TOML config file. Defaults to /etc/sentry-agent/agent.toml - /// if present, otherwise built-in defaults (journald source, default - /// TLS cert paths under /etc/sentry-agent/). + /// Path to a TOML config file. Defaults to the platform's conventional + /// path if present, otherwise built-in defaults — see config::Config::load. #[arg(long)] config: Option, + + #[cfg(windows)] + #[command(subcommand)] + command: Option, } -#[tokio::main] -async fn main() -> Result<()> { +#[cfg(windows)] +#[derive(clap::Subcommand)] +enum WindowsCommand { + /// Registers this binary as a Windows service (Automatic start, + /// LocalSystem account). Requires an administrator shell. + Install, + /// Removes the Windows service registration. + Uninstall, + /// Entry point the Service Control Manager invokes when starting the + /// registered service. Not meant to be run directly by a user — use + /// `sentry-agent` with no subcommand for a normal foreground/console + /// run, same as on Linux. + RunService, +} + +/// Not `#[tokio::main]`: the Windows service dispatcher +/// (`service_dispatcher::start`, see service.rs) is a blocking, synchronous +/// FFI call into the Service Control Manager and needs to be invoked +/// directly from a plain thread, not from inside an already-running tokio +/// runtime. Every other path builds its own runtime explicitly instead. +fn main() -> Result<()> { + let cli = Cli::parse(); + + #[cfg(windows)] + { + match cli.command { + Some(WindowsCommand::Install) => return service::install().context("installing Windows service"), + Some(WindowsCommand::Uninstall) => return service::uninstall().context("removing Windows service"), + Some(WindowsCommand::RunService) => return service::run_as_service().context("running as a Windows service"), + None => {} + } + } + + let rt = tokio::runtime::Runtime::new().context("building tokio runtime")?; + rt.block_on(run_agent(cli.config)) +} + +/// The actual agent: load config, connect to ingest, run the source -> +/// parse -> batch -> ship loop until the source exits or the process is +/// signaled to stop. Called from `main()` directly for a normal run, and +/// from within the Windows service's own thread when running as a +/// service (see service.rs) — same logic either way. +pub async fn run_agent(config_path: Option) -> Result<()> { tracing_subscriber::fmt() .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) .init(); - let cli = Cli::parse(); - let cfg = Config::load(cli.config.as_deref()).context("loading config")?; + let cfg = Config::load(config_path.as_deref()).context("loading config")?; let host = cfg.agent.host.clone().unwrap_or_else(default_hostname); let service = cfg.agent.service.clone(); @@ -60,13 +106,23 @@ async fn main() -> Result<()> { }; let parsed = sentry_parser::parse(&raw.line); let severity = to_pb_severity(raw.severity_hint.or(parsed.severity)); + let mut attributes: std::collections::HashMap = + parsed.attributes.into_iter().collect(); + // Source-provided structured fields (e.g. Windows Event + // Log's EventID/Provider/Channel) win over anything the + // RFC 5424 parser inferred from the raw text, since they + // come from a more authoritative place. + attributes.extend(raw.extra_attributes); let record = LogRecord { timestamp_unix_nano: raw.timestamp_unix_nano, host: host.clone(), service: service.clone(), severity: severity as i32, message: parsed.message, - attributes: parsed.attributes.into_iter().collect(), + attributes, + // Always empty as sent by the agent -- ingest assigns + // this server-side. See the proto field comment. + record_id: String::new(), }; if let Some(batch) = batcher.push(record) { flush(&mut client, batch).await; @@ -87,13 +143,24 @@ async fn main() -> Result<()> { Ok(()) } +// `tx` genuinely goes unused in one rare-but-valid combination: Windows +// features enabled while targeting a non-Windows platform (e.g. sanity- +// checking the Windows source arms compile shape from Linux, which is +// exactly how these were checked before a real Windows toolchain was +// available) collapses every arm to the tx-free `Err(...)` fallback. +#[allow(unused_variables)] async fn spawn_source(source: config::SourceConfig, tx: source::LineSender) { - let result = match source { - #[cfg(feature = "journald")] + // Explicit type: with an unusual feature combination (e.g. + // windows-eventlog enabled while targeting Linux), every arm below can + // collapse to the same untyped `Err(...)` fallback, and Rust can't + // infer the Ok type without at least one real `.await` call anywhere + // in the compiled match to anchor it. + let result: Result<(), anyhow::Error> = match source { + #[cfg(all(feature = "journald", target_os = "linux"))] config::SourceConfig::Journald { unit } => source::journald::run(unit.as_deref(), tx).await, - #[cfg(not(feature = "journald"))] + #[cfg(not(all(feature = "journald", target_os = "linux")))] config::SourceConfig::Journald { .. } => { - Err(anyhow::anyhow!("this build was compiled without the `journald` feature")) + Err(anyhow::anyhow!("this build was compiled without the `journald` feature (or isn't targeting Linux)")) } #[cfg(feature = "file-tail")] @@ -104,6 +171,20 @@ async fn spawn_source(source: config::SourceConfig, tx: source::LineSender) { config::SourceConfig::File { .. } => { Err(anyhow::anyhow!("this build was compiled without the `file-tail` feature")) } + + #[cfg(all(feature = "windows-eventlog", target_os = "windows"))] + config::SourceConfig::EventLog { channels } => source::windows_eventlog::run(&channels, tx).await, + #[cfg(not(all(feature = "windows-eventlog", target_os = "windows")))] + config::SourceConfig::EventLog { .. } => { + Err(anyhow::anyhow!("this build was compiled without the `windows-eventlog` feature (or isn't targeting Windows)")) + } + + #[cfg(all(feature = "etw", target_os = "windows"))] + config::SourceConfig::Etw { providers } => source::etw::run(&providers, tx).await, + #[cfg(not(all(feature = "etw", target_os = "windows")))] + config::SourceConfig::Etw { .. } => { + Err(anyhow::anyhow!("this build was compiled without the `etw` feature (or isn't targeting Windows)")) + } }; if let Err(e) = result { tracing::error!(error = %e, "log source exited with error"); @@ -142,6 +223,7 @@ fn to_pb_severity(sev: Option) -> Severity { } } +#[cfg(not(windows))] fn default_hostname() -> String { if let Ok(s) = std::fs::read_to_string("/etc/hostname") { let s = s.trim().to_string(); @@ -151,3 +233,11 @@ fn default_hostname() -> String { } std::env::var("HOSTNAME").unwrap_or_else(|_| "unknown-host".to_string()) } + +#[cfg(windows)] +fn default_hostname() -> String { + // Windows sets this in every process's environment; no Win32 API call + // needed (GetComputerNameW would be the "proper" way, but this is the + // same value and far simpler). + std::env::var("COMPUTERNAME").unwrap_or_else(|_| "unknown-host".to_string()) +} diff --git a/agent/sentry-agent/src/service.rs b/agent/sentry-agent/src/service.rs new file mode 100644 index 0000000..463ac7e --- /dev/null +++ b/agent/sentry-agent/src/service.rs @@ -0,0 +1,170 @@ +//! Windows Service Control Manager integration: install/uninstall the +//! agent as a native Windows service, and the SCM-invoked entry point +//! that actually runs it as one. +//! +//! UNVERIFIED, same caveat as source/windows_eventlog.rs and +//! source/etw.rs -- written against the `windows-service` crate's +//! documented usage pattern, not compiled or run (no Windows toolchain +//! available). This one is lower-risk than etw.rs (no raw FFI struct +//! layout to get right; `windows-service` wraps that), but the +//! stop-signal plumbing between the SCM callback and the tokio-running +//! agent thread is new code worth testing carefully. +//! +//! Known limitation, not addressed here: when running as a service (no +//! console attached), `tracing_subscriber::fmt()`'s stdout writer has +//! nowhere to go. Logs won't be visible anywhere useful until this is +//! redirected to a file or an actual Windows Event Log tracing sink is +//! written -- flagging this now rather than shipping it silently broken. + +use anyhow::{Context, Result}; +use std::ffi::OsString; +use std::time::Duration; +use windows_service::service::{ + ServiceAccess, ServiceControl, ServiceControlAccept, ServiceErrorControl, ServiceExitCode, + ServiceInfo, ServiceStartType, ServiceState, ServiceStatus, ServiceType, +}; +use windows_service::service_control_handler::{self, ServiceControlHandlerResult}; +use windows_service::service_manager::{ServiceManager, ServiceManagerAccess}; +use windows_service::{define_windows_service, service_dispatcher}; + +pub const SERVICE_NAME: &str = "SentryAgent"; +const SERVICE_TYPE: ServiceType = ServiceType::OWN_PROCESS; + +/// Registers this binary as a Windows service: Automatic start, +/// LocalSystem account, invoked with the `run-service` subcommand (which +/// is what the SCM actually launches — not a bare `sentry-agent` with no +/// arguments). Requires an administrator shell. +pub fn install() -> Result<()> { + let manager = ServiceManager::local_computer(None::<&str>, ServiceManagerAccess::CREATE_SERVICE) + .context("opening Service Control Manager")?; + + let exe_path = std::env::current_exe().context("resolving current executable path")?; + + let service_info = ServiceInfo { + name: OsString::from(SERVICE_NAME), + display_name: OsString::from("Sentry Log Agent"), + service_type: SERVICE_TYPE, + start_type: ServiceStartType::AutoStart, + error_control: ServiceErrorControl::Normal, + executable_path: exe_path, + launch_arguments: vec![OsString::from("run-service")], + dependencies: vec![], + account_name: None, // LocalSystem + account_password: None, + }; + + let service = manager + .create_service(&service_info, ServiceAccess::CHANGE_CONFIG) + .context("creating service")?; + service + .set_description("Ships local logs to Sentry ingest over mTLS.") + .context("setting service description")?; + + tracing::info!(service = SERVICE_NAME, "installed Windows service"); + Ok(()) +} + +/// Removes the service registration. Does not stop a currently-running +/// instance first — stop it via `services.msc`/`sc.exe stop` before +/// uninstalling if it's running. +pub fn uninstall() -> Result<()> { + let manager = ServiceManager::local_computer(None::<&str>, ServiceManagerAccess::CONNECT) + .context("opening Service Control Manager")?; + let service = manager + .open_service(SERVICE_NAME, ServiceAccess::DELETE) + .context("opening service for deletion")?; + service.delete().context("deleting service")?; + + tracing::info!(service = SERVICE_NAME, "removed Windows service"); + Ok(()) +} + +define_windows_service!(ffi_service_main, service_main); + +/// Blocks, handing control to the SCM dispatch loop -- this is what +/// `main()` calls for the `run-service` subcommand, which is what the SCM +/// itself launches when the service starts. Must not be called from +/// inside a tokio runtime (see the doc comment on `main()` in main.rs). +pub fn run_as_service() -> Result<()> { + service_dispatcher::start(SERVICE_NAME, ffi_service_main) + .context("starting Windows service dispatcher") +} + +fn service_main(_arguments: Vec) { + if let Err(e) = run_service() { + // Nowhere better to put this yet -- see the module-level caveat + // about tracing having no attached console under the SCM. + tracing::error!(error = ?e, "windows service run failed"); + } +} + +fn run_service() -> Result<()> { + let (shutdown_tx, shutdown_rx) = std::sync::mpsc::channel::<()>(); + + let event_handler = move |control_event| -> ServiceControlHandlerResult { + match control_event { + ServiceControl::Interrogate => ServiceControlHandlerResult::NoError, + ServiceControl::Stop => { + let _ = shutdown_tx.send(()); + ServiceControlHandlerResult::NoError + } + _ => ServiceControlHandlerResult::NotImplemented, + } + }; + + let status_handle = service_control_handler::register(SERVICE_NAME, event_handler) + .context("registering service control handler")?; + + status_handle + .set_service_status(ServiceStatus { + service_type: SERVICE_TYPE, + current_state: ServiceState::Running, + controls_accepted: ServiceControlAccept::STOP, + exit_code: ServiceExitCode::Win32(0), + checkpoint: 0, + wait_hint: Duration::default(), + process_id: None, + }) + .context("reporting Running status to the SCM")?; + + // service_main is invoked by the SCM on a plain thread, not an async + // context -- build a dedicated tokio runtime here and run the actual + // agent on it, same `run_agent` entry point a normal foreground run + // uses. Block this thread until either the agent exits on its own or + // the SCM asks us to stop. + let agent_thread = std::thread::spawn(|| { + let rt = match tokio::runtime::Runtime::new() { + Ok(rt) => rt, + Err(e) => { + tracing::error!(error = %e, "building tokio runtime for service run"); + return; + } + }; + if let Err(e) = rt.block_on(crate::run_agent(None)) { + tracing::error!(error = %e, "agent exited with error while running as a service"); + } + }); + + loop { + if shutdown_rx.recv_timeout(Duration::from_millis(500)).is_ok() { + break; + } + if agent_thread.is_finished() { + break; + } + } + + status_handle + .set_service_status(ServiceStatus { + service_type: SERVICE_TYPE, + current_state: ServiceState::Stopped, + controls_accepted: ServiceControlAccept::empty(), + exit_code: ServiceExitCode::Win32(0), + checkpoint: 0, + wait_hint: Duration::default(), + process_id: None, + }) + .context("reporting Stopped status to the SCM")?; + + Ok(()) +} diff --git a/agent/sentry-agent/src/source/etw.rs b/agent/sentry-agent/src/source/etw.rs new file mode 100644 index 0000000..ac0401c --- /dev/null +++ b/agent/sentry-agent/src/source/etw.rs @@ -0,0 +1,248 @@ +//! ETW (Event Tracing for Windows) source: a real-time trace session +//! subscribed to specific provider GUIDs. +//! +//! UNVERIFIED, and the highest-risk file in this whole Windows integration +//! -- more so than windows_eventlog.rs. `EVENT_TRACE_PROPERTIES` requires +//! a variable-length buffer appended after the fixed struct (a classic C +//! "flexible array member" pattern for LoggerName), which is exactly the +//! kind of FFI layout detail most likely to be subtly wrong without a +//! Windows toolchain to actually compile and run this against. No Windows +//! target was available in the environment this was written in -- see the +//! module-level note in windows_eventlog.rs for what that means. Compile- +//! check and test this file specifically, first, before trusting any of +//! it. +//! +//! Providers are configured by **GUID**, not friendly name (e.g. +//! `"{22FB2CD6-0E7B-422B-A0C7-2FAD1FD0E716}"`) -- ETW's own +//! `EnableTraceEx2` API takes a GUID, not a name, and there's no simple +//! name-to-GUID resolution in the raw ETW API (that needs the separate TDH +//! provider-enumeration API, not implemented here). Look up a provider's +//! GUID with `logman query providers ""`. +//! +//! Message extraction here is deliberately limited to what's available +//! directly on `EVENT_RECORD`'s header (ProviderId, EventID, Level, +//! Keywords, timestamp, process/thread ID) -- no TDH-based property +//! decoding or message-template rendering (`TdhGetEventInformation`), +//! which is a meaningfully larger undertaking left for a follow-up. This +//! gives real session/provider/callback plumbing with a coarse message, +//! not full structured event decoding. + +use super::{LineSender, RawLine}; +use anyhow::{Context, Result}; +use std::collections::HashMap; +use std::ffi::c_void; +use std::sync::mpsc as std_mpsc; +use std::time::{SystemTime, UNIX_EPOCH}; +use tokio::sync::mpsc as tokio_mpsc; + +use windows::core::{GUID, PCWSTR}; +use windows::Win32::System::Diagnostics::Etw::{ + CloseTrace, ControlTraceW, EnableTraceEx2, OpenTraceW, ProcessTrace, StartTraceW, + EVENT_CONTROL_CODE_ENABLE_PROVIDER, EVENT_RECORD, EVENT_TRACE_CONTROL_STOP, + EVENT_TRACE_LOGFILEW, EVENT_TRACE_LOGFILEW_0, EVENT_TRACE_LOGFILEW_1, + EVENT_TRACE_PROPERTIES, EVENT_TRACE_REAL_TIME_MODE, PROCESS_TRACE_MODE_EVENT_RECORD, + PROCESS_TRACE_MODE_REAL_TIME, TRACE_LEVEL_VERBOSE, +}; + +const SESSION_NAME: &str = "SentryAgentEtw"; + +pub async fn run(providers: &[String], tx: LineSender) -> Result<()> { + let providers = providers.to_vec(); + let (blocking_tx, mut blocking_rx) = tokio_mpsc::channel::(256); + + let handle = tokio::task::spawn_blocking(move || run_session(&providers, blocking_tx)); + + while let Some(line) = blocking_rx.recv().await { + if tx.send(line).await.is_err() { + break; + } + } + + handle.await.context("ETW session task panicked")??; + Ok(()) +} + +fn to_wide(s: &str) -> Vec { + s.encode_utf16().chain(std::iter::once(0)).collect() +} + +/// Thread-local-ish channel used to get the async sender into the +/// C-callable `event_record_callback`, which has a fixed extern "system" +/// signature and can't capture a closure. Set once per `run_session` call +/// before `ProcessTrace` starts invoking the callback. +thread_local! { + static CALLBACK_TX: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; +} + +fn run_session(providers: &[String], tx: tokio_mpsc::Sender) -> Result<()> { + let guids: Vec = providers + .iter() + .map(|p| GUID::try_from(p.as_str()).with_context(|| format!("invalid provider GUID: {p}"))) + .collect::>()?; + + unsafe { + let session_handle = start_session()?; + for guid in &guids { + enable_provider(session_handle, guid)?; + } + + // ProcessTrace runs the consumer loop on *this* thread until + // CloseTrace is called (from the callback, or from another + // thread against the same handle) -- there's no separate + // "shutdown channel" here because the agent's top-level shutdown + // path currently aborts the whole spawn_blocking task rather + // than signaling sources to stop gracefully (same as the other + // sources today). + CALLBACK_TX.with(|cell| *cell.borrow_mut() = Some(tx)); + + let mut logfile = EVENT_TRACE_LOGFILEW::default(); + let mut session_name_wide = to_wide(SESSION_NAME); + logfile.LoggerName = PCWSTR(session_name_wide.as_mut_ptr()); + logfile.Anonymous1 = EVENT_TRACE_LOGFILEW_0 { + ProcessTraceMode: PROCESS_TRACE_MODE_REAL_TIME.0 | PROCESS_TRACE_MODE_EVENT_RECORD.0, + }; + logfile.Anonymous2 = EVENT_TRACE_LOGFILEW_1 { + EventRecordCallback: Some(event_record_callback), + }; + + let trace_handle = OpenTraceW(&mut logfile); + if trace_handle.0 == u64::MAX as usize { + anyhow::bail!("OpenTraceW failed"); + } + + let result = ProcessTrace(&[trace_handle], None, None); + + let _ = CloseTrace(trace_handle); + stop_session(session_handle); + + result.ok().context("ProcessTrace failed")?; + } + + Ok(()) +} + +unsafe fn start_session() -> Result { + // EVENT_TRACE_PROPERTIES needs a trailing buffer (appended after the + // fixed struct) for the session's LoggerName -- this is the flexible- + // array-member pattern flagged in the module doc comment as the + // highest-risk detail in this file. LogFileNameOffset is left 0 (no + // log file; real-time only). + const LOGGER_NAME_CAPACITY: usize = 256; + let total_size = std::mem::size_of::() + LOGGER_NAME_CAPACITY * 2; + let mut buffer = vec![0u8; total_size]; + + let props = buffer.as_mut_ptr() as *mut EVENT_TRACE_PROPERTIES; + (*props).Wnode.BufferSize = total_size as u32; + (*props).Wnode.Flags = windows::Win32::System::Diagnostics::Etw::WNODE_FLAG_TRACED_GUID; + (*props).LogFileMode = EVENT_TRACE_REAL_TIME_MODE; + (*props).LoggerNameOffset = std::mem::size_of::() as u32; + + let session_name_wide = to_wide(SESSION_NAME); + let mut session_handle = Default::default(); + StartTraceW( + &mut session_handle, + PCWSTR(session_name_wide.as_ptr()), + props, + ) + .ok() + .context("StartTraceW failed")?; + + Ok(session_handle) +} + +unsafe fn enable_provider( + session_handle: windows::Win32::System::Diagnostics::Etw::CONTROLTRACE_HANDLE, + guid: &GUID, +) -> Result<()> { + EnableTraceEx2( + session_handle, + guid, + EVENT_CONTROL_CODE_ENABLE_PROVIDER.0, + TRACE_LEVEL_VERBOSE as u8, + 0, + 0, + 0, + None, + ) + .ok() + .with_context(|| format!("EnableTraceEx2 failed for provider {guid:?}")) +} + +unsafe fn stop_session(session_handle: windows::Win32::System::Diagnostics::Etw::CONTROLTRACE_HANDLE) { + let mut buffer = vec![0u8; std::mem::size_of::() + 512]; + let props = buffer.as_mut_ptr() as *mut EVENT_TRACE_PROPERTIES; + (*props).Wnode.BufferSize = buffer.len() as u32; + let _ = ControlTraceW(session_handle, PCWSTR::null(), props, EVENT_TRACE_CONTROL_STOP); +} + +/// `extern "system"` callback ETW invokes per event during `ProcessTrace`. +/// Deliberately minimal: header fields only, no TDH property decoding +/// (see module doc comment). +unsafe extern "system" fn event_record_callback(record: *mut EVENT_RECORD) { + if record.is_null() { + return; + } + let record = &*record; + let header = &record.EventHeader; + + let mut attributes = HashMap::new(); + attributes.insert( + "etw.provider_guid".to_string(), + format!("{:?}", header.ProviderId), + ); + attributes.insert("etw.event_id".to_string(), header.EventDescriptor.Id.to_string()); + attributes.insert( + "etw.opcode".to_string(), + header.EventDescriptor.Opcode.to_string(), + ); + attributes.insert( + "etw.task".to_string(), + header.EventDescriptor.Task.to_string(), + ); + attributes.insert("etw.process_id".to_string(), header.ProcessId.to_string()); + attributes.insert("etw.thread_id".to_string(), header.ThreadId.to_string()); + + let severity_hint = etw_level_to_syslog_severity(header.EventDescriptor.Level); + + let timestamp_unix_nano = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos() as i64) + .unwrap_or(0); + + // No TDH-based message rendering (see module doc comment) -- this is + // a coarse, structured summary rather than a human-authored message. + // Downstream (sentry_parser's raw-passthrough fallback) handles a + // non-RFC5424 line like this the same as any other raw line. + let message = format!( + "ETW event: provider={:?} id={} level={}", + header.ProviderId, header.EventDescriptor.Id, header.EventDescriptor.Level + ); + + let raw = RawLine { + line: message, + timestamp_unix_nano, + severity_hint, + extra_attributes: attributes, + }; + + CALLBACK_TX.with(|cell| { + if let Some(tx) = cell.borrow().as_ref() { + let _ = tx.blocking_send(raw); + } + }); +} + +/// Maps ETW `Level` (0=LogAlways/Verbose-ish through 5=Verbose, following +/// the same TRACE_LEVEL_* scale as windows_eventlog's Level values) onto +/// the syslog 0-7 scale, same reasoning as windows_eventlog.rs. +fn etw_level_to_syslog_severity(level: u8) -> Option { + match level { + 1 => Some(2), // Critical -> crit + 2 => Some(3), // Error -> err + 3 => Some(4), // Warning -> warning + 4 => Some(6), // Informational -> info + 5 => Some(7), // Verbose -> debug + _ => None, + } +} diff --git a/agent/sentry-agent/src/source/file_tail.rs b/agent/sentry-agent/src/source/file_tail.rs index f4378a6..fa7742e 100644 --- a/agent/sentry-agent/src/source/file_tail.rs +++ b/agent/sentry-agent/src/source/file_tail.rs @@ -61,6 +61,7 @@ pub async fn run(path: &Path, from_beginning: bool, tx: LineSender) -> Result<() line, timestamp_unix_nano, severity_hint: None, + extra_attributes: Default::default(), }) .await .is_err() diff --git a/agent/sentry-agent/src/source/journald.rs b/agent/sentry-agent/src/source/journald.rs index d988845..98fdf3e 100644 --- a/agent/sentry-agent/src/source/journald.rs +++ b/agent/sentry-agent/src/source/journald.rs @@ -61,6 +61,7 @@ pub async fn run(unit: Option<&str>, tx: LineSender) -> Result<()> { line: message, timestamp_unix_nano, severity_hint, + extra_attributes: Default::default(), }) .await .is_err() diff --git a/agent/sentry-agent/src/source/mod.rs b/agent/sentry-agent/src/source/mod.rs index ae52b84..0942dc3 100644 --- a/agent/sentry-agent/src/source/mod.rs +++ b/agent/sentry-agent/src/source/mod.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use tokio::sync::mpsc; /// A raw line read from a source, plus whatever metadata the source itself @@ -8,16 +9,31 @@ pub struct RawLine { /// Unix epoch nanoseconds at time of read. pub timestamp_unix_nano: i64, /// Syslog severity (0-7) if the source already knows it independent of - /// the line's own content — e.g. journald's PRIORITY field. When set, - /// this takes precedence over whatever the RFC 5424 parser infers from - /// the message text, since it comes from a more authoritative place. + /// the line's own content — e.g. journald's PRIORITY field, or a + /// Windows Event Log Level mapped onto the same scale. When set, this + /// takes precedence over whatever the RFC 5424 parser infers from the + /// message text, since it comes from a more authoritative place. pub severity_hint: Option, + /// Structured fields the source already knows, independent of the raw + /// message text — e.g. Windows Event Log's EventID/Provider/Channel. + /// Merged into the record's attributes alongside whatever the RFC 5424 + /// parser extracts from `line`; on key collision, these win, since + /// they also come from a more authoritative place than text parsing. + /// Sources that have nothing to add (journald, file-tail) just leave + /// this empty. + pub extra_attributes: HashMap, } pub type LineSender = mpsc::Sender; -#[cfg(feature = "journald")] +#[cfg(all(feature = "journald", target_os = "linux"))] pub mod journald; #[cfg(feature = "file-tail")] pub mod file_tail; + +#[cfg(all(feature = "windows-eventlog", target_os = "windows"))] +pub mod windows_eventlog; + +#[cfg(all(feature = "etw", target_os = "windows"))] +pub mod etw; diff --git a/agent/sentry-agent/src/source/windows_eventlog.rs b/agent/sentry-agent/src/source/windows_eventlog.rs new file mode 100644 index 0000000..d94789b --- /dev/null +++ b/agent/sentry-agent/src/source/windows_eventlog.rs @@ -0,0 +1,283 @@ +//! Windows Event Log source via `EvtSubscribe`. +//! +//! UNVERIFIED: this module was written against the documented +//! EvtSubscribe/EvtNext/EvtRender API shape (the same pull-model pattern +//! Microsoft's own C++ samples use for subscriptions), but has not been +//! compiled or run on a real Windows host — no Windows target toolchain +//! was available in the environment this was written in (confirmed: only +//! x86_64-unknown-linux-gnu std was installed, no rustup, no way to even +//! `cargo check --target x86_64-pc-windows-*`). Treat this as a first +//! draft to compile-check and test for real before trusting it. See +//! /docs/phase-1-runbook.md for what's actually been verified vs. not. + +use super::{LineSender, RawLine}; +use anyhow::{Context, Result}; +use std::collections::HashMap; +use std::time::{SystemTime, UNIX_EPOCH}; +use tokio::sync::mpsc as tokio_mpsc; + +use windows::core::PCWSTR; +use windows::Win32::Foundation::{ERROR_NO_MORE_ITEMS, WAIT_OBJECT_0}; +use windows::Win32::System::EventLog::{ + EvtClose, EvtNext, EvtRender, EvtRenderEventXml, EvtSubscribe, EVT_HANDLE, + EVT_SUBSCRIBE_TO_FUTURE_EVENTS, +}; +use windows::Win32::System::Threading::{CreateEventW, WaitForSingleObject}; + +/// Tails one or more Windows Event Log channels. Runs the blocking +/// EvtSubscribe/EvtNext calls on a dedicated OS thread per channel (via +/// `spawn_blocking`) and forwards parsed lines back over `tx`, same shape +/// as the journald source's subprocess-reading loop. +pub async fn run(channels: &[String], tx: LineSender) -> Result<()> { + let channels = channels.to_vec(); + let (blocking_tx, mut blocking_rx) = tokio_mpsc::channel::(256); + + let handle = tokio::task::spawn_blocking(move || subscribe_all(&channels, blocking_tx)); + + while let Some(line) = blocking_rx.recv().await { + if tx.send(line).await.is_err() { + break; // receiver dropped, agent is shutting down + } + } + + handle + .await + .context("windows event log subscription task panicked")??; + Ok(()) +} + +/// One `std::thread` per channel, each blocked in its own +/// wait-then-drain loop. Simpler and still correct for the common case of +/// 1-3 channels; a single `WaitForMultipleObjects`-based dispatcher would +/// scale better to many channels but isn't needed for Phase 1's default +/// three (Application/System/Security). +fn subscribe_all(channels: &[String], tx: tokio_mpsc::Sender) -> Result<()> { + let mut threads = Vec::with_capacity(channels.len()); + for channel in channels { + let channel = channel.clone(); + let tx = tx.clone(); + threads.push(std::thread::spawn(move || subscribe_one(&channel, tx))); + } + for t in threads { + t.join() + .map_err(|_| anyhow::anyhow!("event log subscriber thread panicked"))??; + } + Ok(()) +} + +fn to_wide(s: &str) -> Vec { + s.encode_utf16().chain(std::iter::once(0)).collect() +} + +fn subscribe_one(channel: &str, tx: tokio_mpsc::Sender) -> Result<()> { + unsafe { + let signal_event = CreateEventW(None, true, false, None) + .context("CreateEventW for subscription signal failed")?; + + let channel_wide = to_wide(channel); + // NULL query (PCWSTR::null()) means "all events on this channel". + // No callback (None) -- pull model via the signal event instead, + // so this stays a plain loop rather than a Win32 callback that + // would need to cross back into the tokio runtime. + let subscription = EvtSubscribe( + None, + Some(signal_event), + PCWSTR(channel_wide.as_ptr()), + PCWSTR::null(), + None, + None, + None, + EVT_SUBSCRIBE_TO_FUTURE_EVENTS.0, + ) + .context("EvtSubscribe failed")?; + + loop { + let wait = WaitForSingleObject(signal_event, u32::MAX); + if wait != WAIT_OBJECT_0 { + anyhow::bail!("WaitForSingleObject on event log subscription failed"); + } + + loop { + let mut events: [EVT_HANDLE; 16] = [EVT_HANDLE::default(); 16]; + let mut returned = 0u32; + let next = EvtNext(subscription, &mut events, 0, 0, &mut returned); + if let Err(err) = next { + if err.code() == ERROR_NO_MORE_ITEMS.into() { + break; // drained this batch; go back to waiting on the signal + } + return Err(err).context("EvtNext failed"); + } + + for &event in &events[..returned as usize] { + if let Some(raw) = render_event(event, channel) { + if tx.blocking_send(raw).is_err() { + let _ = EvtClose(event); + let _ = EvtClose(subscription); + return Ok(()); + } + } + let _ = EvtClose(event); + } + } + } + } +} + +fn render_event(event: EVT_HANDLE, channel: &str) -> Option { + unsafe { + let mut buffer_used = 0u32; + let mut property_count = 0u32; + // First call with a zero-length buffer to learn the required size. + let _ = EvtRender( + None, + event, + EvtRenderEventXml.0, + 0, + None, + &mut buffer_used, + &mut property_count, + ); + if buffer_used == 0 { + return None; + } + + let mut buffer = vec![0u16; (buffer_used as usize).div_ceil(2)]; + let rendered = EvtRender( + None, + event, + EvtRenderEventXml.0, + buffer_used, + Some(buffer.as_mut_ptr() as *mut _), + &mut buffer_used, + &mut property_count, + ); + if rendered.is_err() { + return None; + } + + let xml = String::from_utf16_lossy(&buffer); + let xml = xml.trim_end_matches('\0'); + parse_event_xml(xml, channel) + } +} + +/// Minimal, deliberately non-validating extraction of the fields Phase 1 +/// needs from the rendered event XML: EventID, Provider, Level, Computer, +/// Windows' own EventRecordID, and a best-effort message. Not a full XML +/// parser in the schema-aware sense -- uses `quick-xml`'s streaming +/// reader to pull specific elements/attributes rather than hand-rolled +/// string search, but doesn't attempt full EventData/UserData schema +/// awareness across every provider's custom shape. Worth revisiting once +/// this is running against real events from real providers. +fn parse_event_xml(xml: &str, channel: &str) -> Option { + use quick_xml::events::Event as XmlEvent; + use quick_xml::reader::Reader; + + let mut reader = Reader::from_str(xml); + reader.config_mut().trim_text(true); + + let mut event_id = None; + let mut provider = None; + let mut level = None; + let mut computer = None; + let mut record_id = None; + let mut event_data_values: Vec = Vec::new(); + + let mut current_tag: Option = None; + let mut buf = Vec::new(); + + loop { + match reader.read_event_into(&mut buf) { + Ok(XmlEvent::Start(e)) | Ok(XmlEvent::Empty(e)) => { + let name = local_name(&e); + if name == "Provider" { + for attr in e.attributes().flatten() { + if attr.key.as_ref() == b"Name" { + provider = attr + .decode_and_unescape_value(reader.decoder()) + .ok() + .map(|v| v.into_owned()); + } + } + } + current_tag = Some(name); + } + Ok(XmlEvent::Text(t)) => { + let text = t.unescape().unwrap_or_default().into_owned(); + match current_tag.as_deref() { + Some("EventID") => event_id = Some(text), + Some("Level") => level = text.parse::().ok(), + Some("Computer") => computer = Some(text), + Some("EventRecordID") => record_id = Some(text), + Some("Data") => event_data_values.push(text), + _ => {} + } + } + Ok(XmlEvent::Eof) => break, + Err(_) => return None, + _ => {} + } + buf.clear(); + } + + // commonly holds one or more value + // elements rather than a single free-text message; joining them is a + // reasonable Phase 1 default until per-provider message templates are + // rendered properly. Real message-template rendering needs + // EvtFormatMessage against the provider's message-table resource -- + // worth a follow-up, not required for a raw-passthrough-shaped record + // (sentry_parser's raw fallback handles this fine either way). + let message = if event_data_values.is_empty() { + xml.to_string() + } else { + event_data_values.join(" | ") + }; + + let mut attributes = HashMap::new(); + if let Some(id) = &event_id { + attributes.insert("winevt.event_id".to_string(), id.clone()); + } + if let Some(p) = provider { + attributes.insert("winevt.provider".to_string(), p); + } + attributes.insert("winevt.channel".to_string(), channel.to_string()); + if let Some(c) = computer { + attributes.insert("winevt.computer".to_string(), c); + } + if let Some(r) = record_id { + attributes.insert("winevt.record_number".to_string(), r); + } + + let severity_hint = level.and_then(windows_level_to_syslog_severity); + + let timestamp_unix_nano = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos() as i64) + .unwrap_or(0); + + Some(RawLine { + line: message, + timestamp_unix_nano, + severity_hint, + extra_attributes: attributes, + }) +} + +fn local_name(e: &quick_xml::events::BytesStart) -> String { + String::from_utf8_lossy(e.local_name().as_ref()).into_owned() +} + +/// Maps Windows Event Log `Level` values (0=LogAlways, 1=Critical, +/// 2=Error, 3=Warning, 4=Informational, 5=Verbose) onto the same syslog +/// 0-7 severity scale `severity_hint` uses everywhere else in the agent, +/// so `main.rs`'s `to_pb_severity` needs no Windows-specific knowledge. +fn windows_level_to_syslog_severity(level: u8) -> Option { + match level { + 1 => Some(2), // Critical -> crit + 2 => Some(3), // Error -> err + 3 => Some(4), // Warning -> warning + 4 => Some(6), // Informational -> info + 5 => Some(7), // Verbose -> debug + _ => None, // 0 (LogAlways) or unrecognized -- let the parser decide + } +} diff --git a/api/Dockerfile b/api/Dockerfile index 7ead31f..bf36a90 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -1,8 +1,11 @@ -# Build context must be the repo root (sentry/): +# Build context must be the repo root (sentry/), since this needs both +# api/ and proto/ (api now speaks gRPC to /search, using proto's checked-in +# Go bindings via the `replace` directive in api/go.mod): # docker build -f api/Dockerfile -t sentry-api . FROM golang:1.25-alpine AS builder WORKDIR /src +COPY proto ./proto COPY api ./api WORKDIR /src/api RUN go mod download diff --git a/api/README.md b/api/README.md index 68f44b8..c674f36 100644 --- a/api/README.md +++ b/api/README.md @@ -1,17 +1,19 @@ # api -Sentry's Phase 0 query API: one crude, intentionally placeholder endpoint. +Sentry's query API: two intentionally crude endpoints — raw SQL and +free-text search — that Phase 2's real query layer replaces outright. ## Why plain REST, not gRPC + REST gateway CLAUDE.md pins the control plane to "Go, gRPC + REST gateway." This -service is plain `net/http` instead — a deliberate Phase 0 simplification, -not a change to the pinned stack. Wiring up a `.proto` service, -`google.api.http` annotations, and `protoc-gen-grpc-gateway` codegen for a -single endpoint that Phase 2 replaces outright with a real SPL-like query +service is plain `net/http` instead — a deliberate simplification, not a +change to the pinned stack. Wiring up `.proto` services, +`google.api.http` annotations, and `protoc-gen-grpc-gateway` codegen for +two endpoints that Phase 2 replaces outright with a real SPL-like query layer would be exactly the kind of premature machinery this project's -conventions warn against. Adopt the gRPC+gateway pattern once `/api` grows -a second real, durable endpoint. +conventions warn against. `api` *does* speak gRPC internally though — to +`/search` (see below) — this simplification is specifically about the +public-facing surface, not a blanket avoidance of gRPC. ## Endpoints @@ -20,10 +22,21 @@ a second real, durable endpoint. SELECT-only, single-statement, basic keyword-based injection guarding (see `internal/queryapi/validate.go` for exactly what that does and doesn't catch — it's not a SQL parser). +- `POST /search` — body `{"query": "...", "limit": 100}`, same response + shape as `/query`. Calls `/search`'s `SearchService.Search` gRPC RPC to + resolve the free-text query into matching `record_id`s, then joins + those back against ClickHouse (`SELECT * FROM logs WHERE record_id IN + (...)`) to return full rows — so both endpoints return the same + `{columns, rows}` shape and `/web` can reuse one table component for + both. Every `record_id` is validated as a real UUID before being + embedded in the generated SQL (defense in depth: `record_id`s come from + an internal, trusted service, not raw user input, but a value that + fails to parse as a UUID can't contain SQL-breaking characters either + way). - `GET /healthz` — for docker-compose/k8s liveness checks. -No auth. Not scoped for Phase 0 — don't expose this beyond a trusted -dev/homelab network. +No auth. Not scoped yet — don't expose this beyond a trusted dev/homelab +network. ## Configuration @@ -34,9 +47,15 @@ Environment variables (see `internal/config/config.go`): | `HTTP_LISTEN_ADDR` | `:8080` | | | `CLICKHOUSE_ADDR` | `localhost:9000` | Native protocol port | | `CLICKHOUSE_DATABASE` / `_USERNAME` / `_PASSWORD` | `sentry` / `default` / `` | | -| `QUERY_TIMEOUT_SECONDS` | `30` | Per-request ClickHouse query timeout | +| `SEARCH_GRPC_ADDR` | `localhost:50052` | Must match `/search`'s `GRPC_LISTEN_ADDR` | +| `QUERY_TIMEOUT_SECONDS` | `30` | Per-request timeout, both endpoints | | `CORS_ALLOWED_ORIGIN` | `*` | Wide open by default since there's no auth yet; tighten together | +`searchclient.Dial` connects to `/search` over plain TCP, no TLS — same +trust boundary as `api`'s existing plain-TCP connection to ClickHouse. +mTLS in this project is specifically the agent↔ingest edge boundary, not +every internal hop. + ## Building & testing ```sh @@ -52,12 +71,15 @@ docker build -f api/Dockerfile -t sentry-api . ## Testing notes -`internal/queryapi`'s HTTP handler depends on ClickHouse only through a -one-method `queryExecutor` interface, so routing, validation, JSON -encoding, and error-status mapping are all unit-tested against a fake — -no live ClickHouse needed. `Executor` itself (the reflection-based row -scanning against `driver.Rows`) is not unit-tested — faking ClickHouse's -`driver.Rows` interface fully would be significant test-only scaffolding -for a Phase 0 placeholder, and the driver package's own docs note it isn't -meant to be implemented by adopters. It's exercised end-to-end via the -docker-compose flow in `/docs/phase-0-runbook.md` instead. +`internal/queryapi`'s HTTP handlers depend on ClickHouse and `/search` +only through narrow interfaces (`queryExecutor`, `searchClient`), so +routing, validation, JSON encoding, error-status mapping, and the +record_id-to-SQL query building are all unit-tested against fakes — no +live ClickHouse or `/search` instance needed. `Executor` itself (the +reflection-based row scanning against `driver.Rows`) and +`internal/searchclient`'s actual gRPC dial are not unit-tested — the +former because faking ClickHouse's `driver.Rows` interface fully would be +significant test-only scaffolding the driver's own docs say isn't meant +to be implemented by adopters; the latter because it's a thin wrapper +with nothing but wiring to test. Both are exercised end-to-end via the +docker-compose flow in `/docs/phase-1-runbook.md` instead. diff --git a/api/cmd/api/main.go b/api/cmd/api/main.go index 779c10c..04c837d 100644 --- a/api/cmd/api/main.go +++ b/api/cmd/api/main.go @@ -1,7 +1,7 @@ -// Command api is the Sentry Phase 0 query API: a single crude POST /query -// endpoint proxying allowlisted SELECT statements to ClickHouse. See -// internal/queryapi for why this is plain REST rather than the pinned -// gRPC+gateway pattern for Phase 0. +// Command api is Sentry's query API: POST /query (raw SQL, SELECT-only) +// and POST /search (free-text, via the search service). See +// internal/queryapi for why these are plain REST rather than the pinned +// gRPC+gateway pattern. package main import ( @@ -17,6 +17,7 @@ import ( "github.com/sentry/sentry/api/internal/config" "github.com/sentry/sentry/api/internal/queryapi" + "github.com/sentry/sentry/api/internal/searchclient" ) func main() { @@ -50,8 +51,15 @@ func main() { os.Exit(1) } + search, err := searchclient.Dial(cfg.SearchGRPCAddr) + if err != nil { + logger.Error("dialing search service", "error", err) + os.Exit(1) + } + defer search.Close() + exec := queryapi.NewExecutor(conn) - handler := queryapi.NewHandler(logger, exec, cfg.QueryTimeout, cfg.CORSAllowedOrigin) + handler := queryapi.NewHandler(logger, exec, search, cfg.QueryTimeout, cfg.CORSAllowedOrigin) srv := &http.Server{ Addr: cfg.HTTPListenAddr, diff --git a/api/go.mod b/api/go.mod index 731774b..92ddfff 100644 --- a/api/go.mod +++ b/api/go.mod @@ -2,7 +2,14 @@ module github.com/sentry/sentry/api go 1.25.0 -require github.com/ClickHouse/clickhouse-go/v2 v2.48.0 +require ( + github.com/ClickHouse/clickhouse-go/v2 v2.48.0 + github.com/google/uuid v1.6.0 + github.com/sentry/sentry/proto v0.0.0-00010101000000-000000000000 + google.golang.org/grpc v1.83.0 +) + +replace github.com/sentry/sentry/proto => ../proto require ( github.com/ClickHouse/ch-go v0.74.0 // indirect @@ -10,7 +17,6 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/go-faster/city v1.0.1 // indirect github.com/go-faster/errors v0.7.1 // indirect - github.com/google/uuid v1.6.0 // indirect github.com/klauspost/compress v1.19.1 // indirect github.com/paulmach/orb v0.13.0 // indirect github.com/pierrec/lz4/v4 v4.1.27 // indirect @@ -18,5 +24,9 @@ require ( github.com/shopspring/decimal v1.4.0 // indirect go.opentelemetry.io/otel v1.44.0 // indirect go.opentelemetry.io/otel/trace v1.44.0 // indirect + golang.org/x/net v0.57.0 // indirect golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/protobuf v1.36.12 // indirect ) diff --git a/api/go.sum b/api/go.sum index 6bf06f1..4001ba2 100644 --- a/api/go.sum +++ b/api/go.sum @@ -12,6 +12,12 @@ github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -32,11 +38,31 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ= +google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/api/internal/config/config.go b/api/internal/config/config.go index 1408fa5..3a61e62 100644 --- a/api/internal/config/config.go +++ b/api/internal/config/config.go @@ -12,6 +12,7 @@ import ( type Config struct { HTTPListenAddr string ClickHouse ClickHouseConfig + SearchGRPCAddr string QueryTimeout time.Duration CORSAllowedOrigin string } @@ -32,6 +33,9 @@ func Load() (Config, error) { Username: getenv("CLICKHOUSE_USERNAME", "default"), Password: getenv("CLICKHOUSE_PASSWORD", ""), }, + // Search service's gRPC address (see /search) -- default matches + // /search's own default GRPC_LISTEN_ADDR. + SearchGRPCAddr: getenv("SEARCH_GRPC_ADDR", "localhost:50052"), // Phase 0 has no auth, so this is wide open by default to keep // the local SvelteKit dev server (a different origin/port) // working out of the box. Tighten before this is ever reachable diff --git a/api/internal/config/config_test.go b/api/internal/config/config_test.go index 17d6885..8eabb2d 100644 --- a/api/internal/config/config_test.go +++ b/api/internal/config/config_test.go @@ -19,6 +19,9 @@ func TestLoadDefaults(t *testing.T) { if cfg.CORSAllowedOrigin != "*" { t.Errorf("CORSAllowedOrigin = %q, want *", cfg.CORSAllowedOrigin) } + if cfg.SearchGRPCAddr != "localhost:50052" { + t.Errorf("SearchGRPCAddr = %q, want localhost:50052", cfg.SearchGRPCAddr) + } } func TestLoadInvalidTimeoutErrors(t *testing.T) { diff --git a/api/internal/queryapi/handler.go b/api/internal/queryapi/handler.go index 60355e2..93143cc 100644 --- a/api/internal/queryapi/handler.go +++ b/api/internal/queryapi/handler.go @@ -1,13 +1,13 @@ -// Package queryapi is the Phase 0 query API: a single crude POST /query -// endpoint that takes a raw SQL string, allowlists it to a single SELECT -// statement, and proxies it to ClickHouse. This is a deliberate -// simplification of the pinned "gRPC + REST gateway" control-plane -// pattern (see CLAUDE.md's tech stack table): a plain net/http REST -// handler, not a gRPC service transcoded through grpc-gateway. That -// machinery (proto definitions, googleapis annotations, gateway codegen) -// buys nothing for one crude placeholder endpoint that Phase 2 replaces -// outright with the real SPL-like query layer. Revisit gRPC+gateway when -// /api grows a second real endpoint. +// Package queryapi is Sentry's query API: POST /query (Phase 0, a crude +// raw-SQL passthrough allowlisted to SELECT) and POST /search (Phase 1, +// free-text search via the search service, joined back against +// ClickHouse). This is a deliberate simplification of the pinned "gRPC + +// REST gateway" control-plane pattern (see CLAUDE.md's tech stack table): +// plain net/http REST handlers, not a gRPC service transcoded through +// grpc-gateway. That machinery (proto definitions, googleapis +// annotations, gateway codegen) doesn't buy much for two crude endpoints +// that Phase 2's real SPL-like query layer replaces outright. Revisit +// gRPC+gateway once /api's endpoint count and lifespan justify it. package queryapi import ( @@ -28,17 +28,19 @@ type queryExecutor interface { type Handler struct { logger *slog.Logger exec queryExecutor + search searchClient queryTimeout time.Duration allowedOrigin string } -func NewHandler(logger *slog.Logger, exec queryExecutor, queryTimeout time.Duration, allowedOrigin string) *Handler { - return &Handler{logger: logger, exec: exec, queryTimeout: queryTimeout, allowedOrigin: allowedOrigin} +func NewHandler(logger *slog.Logger, exec queryExecutor, search searchClient, queryTimeout time.Duration, allowedOrigin string) *Handler { + return &Handler{logger: logger, exec: exec, search: search, queryTimeout: queryTimeout, allowedOrigin: allowedOrigin} } func (h *Handler) Routes() http.Handler { mux := http.NewServeMux() mux.HandleFunc("POST /query", h.handleQuery) + mux.HandleFunc("POST /search", h.handleSearch) mux.HandleFunc("GET /healthz", h.handleHealthz) return h.withCORS(mux) } @@ -99,10 +101,7 @@ func (h *Handler) handleQuery(w http.ResponseWriter, r *http.Request) { return } - w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(result); err != nil { - h.logger.Error("encoding response", "error", err) - } + writeJSON(w, result) } func writeError(w http.ResponseWriter, status int, msg string) { diff --git a/api/internal/queryapi/handler_test.go b/api/internal/queryapi/handler_test.go index 399c007..82e487f 100644 --- a/api/internal/queryapi/handler_test.go +++ b/api/internal/queryapi/handler_test.go @@ -27,8 +27,24 @@ func (f *fakeExecutor) Execute(_ context.Context, sql string) (*QueryResult, err return f.result, nil } +type fakeSearchClient struct { + recordIDs []string + err error +} + +func (f *fakeSearchClient) Search(_ context.Context, _ string, _ uint32) ([]string, error) { + if f.err != nil { + return nil, f.err + } + return f.recordIDs, nil +} + func newTestHandler(exec queryExecutor) *Handler { - return NewHandler(slog.New(slog.NewTextHandler(io.Discard, nil)), exec, time.Second, "*") + return newTestHandlerWithSearch(exec, &fakeSearchClient{}) +} + +func newTestHandlerWithSearch(exec queryExecutor, search searchClient) *Handler { + return NewHandler(slog.New(slog.NewTextHandler(io.Discard, nil)), exec, search, time.Second, "*") } func TestHandleQuerySuccess(t *testing.T) { diff --git a/api/internal/queryapi/search.go b/api/internal/queryapi/search.go new file mode 100644 index 0000000..40e0d89 --- /dev/null +++ b/api/internal/queryapi/search.go @@ -0,0 +1,96 @@ +package queryapi + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/google/uuid" +) + +// searchClient is the narrow interface handleSearch depends on, so tests +// can substitute a fake without a real search service. A small gRPC +// adapter in cmd/api satisfies this. +type searchClient interface { + Search(ctx context.Context, query string, limit uint32) ([]string, error) +} + +type searchRequest struct { + Query string `json:"query"` + Limit uint32 `json:"limit"` +} + +func (h *Handler) handleSearch(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxBodyBytes) + + var req searchRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "invalid JSON body: "+err.Error()) + return + } + if strings.TrimSpace(req.Query) == "" { + writeError(w, http.StatusBadRequest, "query must not be empty") + return + } + + ctx, cancel := context.WithTimeout(r.Context(), h.queryTimeout) + defer cancel() + + recordIDs, err := h.search.Search(ctx, req.Query, req.Limit) + if err != nil { + h.logger.Error("search failed", "error", err) + writeError(w, http.StatusBadGateway, "search failed: "+err.Error()) + return + } + + if len(recordIDs) == 0 { + writeJSON(w, &QueryResult{Columns: []string{}, Rows: [][]any{}}) + return + } + + sql, err := recordIDsQuery(recordIDs) + if err != nil { + h.logger.Error("building record_id query", "error", err) + writeError(w, http.StatusBadGateway, "search returned unusable results") + return + } + + result, err := h.exec.Execute(ctx, sql) + if err != nil { + h.logger.Error("joining search results against clickhouse failed", "error", err) + writeError(w, http.StatusBadGateway, "query failed: "+err.Error()) + return + } + + writeJSON(w, result) +} + +// recordIDsQuery builds a SELECT ... WHERE record_id IN (...) against the +// IDs the search service returned. Every ID is validated as a real UUID +// before being embedded in the query string -- record_ids come from an +// internal, trusted service (not raw user input), but a UUID that fails +// to parse can't contain SQL-breaking characters either way, so this is +// defense in depth, not a response to a specific threat. +func recordIDsQuery(recordIDs []string) (string, error) { + quoted := make([]string, 0, len(recordIDs)) + for _, id := range recordIDs { + if _, err := uuid.Parse(id); err != nil { + continue // skip anything not a valid UUID rather than failing the whole query + } + quoted = append(quoted, "'"+id+"'") + } + if len(quoted) == 0 { + return "", fmt.Errorf("no valid record_ids in search response") + } + return fmt.Sprintf( + "SELECT * FROM logs WHERE record_id IN (%s) ORDER BY timestamp DESC", + strings.Join(quoted, ","), + ), nil +} + +func writeJSON(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(v) +} diff --git a/api/internal/queryapi/search_test.go b/api/internal/queryapi/search_test.go new file mode 100644 index 0000000..5a76c0e --- /dev/null +++ b/api/internal/queryapi/search_test.go @@ -0,0 +1,125 @@ +package queryapi + +import ( + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestHandleSearchSuccess(t *testing.T) { + id := "5754b062-ec8b-45b1-b1b8-a50f263adcd3" + fe := &fakeExecutor{result: &QueryResult{ + Columns: []string{"message"}, + Rows: [][]any{{"hello world"}}, + }} + fs := &fakeSearchClient{recordIDs: []string{id}} + h := newTestHandlerWithSearch(fe, fs) + + body := strings.NewReader(`{"query": "hello"}`) + req := httptest.NewRequest(http.MethodPost, "/search", body) + rec := httptest.NewRecorder() + + h.Routes().ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + if !strings.Contains(fe.gotSQL, id) { + t.Fatalf("expected the record_id in the generated SQL, got %q", fe.gotSQL) + } + if !strings.Contains(fe.gotSQL, "WHERE record_id IN") { + t.Fatalf("expected an IN clause, got %q", fe.gotSQL) + } + + var got QueryResult + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatalf("decoding response: %v", err) + } + if len(got.Rows) != 1 { + t.Fatalf("unexpected result: %+v", got) + } +} + +func TestHandleSearchRejectsEmptyQuery(t *testing.T) { + fe := &fakeExecutor{} + fs := &fakeSearchClient{} + h := newTestHandlerWithSearch(fe, fs) + + body := strings.NewReader(`{"query": " "}`) + req := httptest.NewRequest(http.MethodPost, "/search", body) + rec := httptest.NewRecorder() + + h.Routes().ServeHTTP(rec, req) + + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400", rec.Code) + } + if fe.gotSQL != "" { + t.Fatal("executor should not have been called for an empty query") + } +} + +func TestHandleSearchNoResultsReturnsEmptyNotError(t *testing.T) { + fe := &fakeExecutor{} + fs := &fakeSearchClient{recordIDs: nil} + h := newTestHandlerWithSearch(fe, fs) + + body := strings.NewReader(`{"query": "nothing matches this"}`) + req := httptest.NewRequest(http.MethodPost, "/search", body) + rec := httptest.NewRecorder() + + h.Routes().ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + if fe.gotSQL != "" { + t.Fatal("executor should not have been called when search returns no IDs") + } + + var got QueryResult + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatalf("decoding response: %v", err) + } + if len(got.Rows) != 0 { + t.Fatalf("expected empty rows, got %+v", got.Rows) + } +} + +func TestHandleSearchServiceErrorReturnsBadGateway(t *testing.T) { + fe := &fakeExecutor{} + fs := &fakeSearchClient{err: errors.New("search service unreachable")} + h := newTestHandlerWithSearch(fe, fs) + + body := strings.NewReader(`{"query": "hello"}`) + req := httptest.NewRequest(http.MethodPost, "/search", body) + rec := httptest.NewRecorder() + + h.Routes().ServeHTTP(rec, req) + + if rec.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want 502", rec.Code) + } +} + +func TestRecordIDsQuerySkipsInvalidUUIDs(t *testing.T) { + sql, err := recordIDsQuery([]string{"not-a-uuid", "5754b062-ec8b-45b1-b1b8-a50f263adcd3"}) + if err != nil { + t.Fatalf("recordIDsQuery() error = %v", err) + } + if strings.Contains(sql, "not-a-uuid") { + t.Fatalf("expected the invalid UUID to be skipped, got %q", sql) + } + if !strings.Contains(sql, "5754b062-ec8b-45b1-b1b8-a50f263adcd3") { + t.Fatalf("expected the valid UUID to be included, got %q", sql) + } +} + +func TestRecordIDsQueryAllInvalidReturnsError(t *testing.T) { + if _, err := recordIDsQuery([]string{"not-a-uuid", "also-not-one"}); err == nil { + t.Fatal("expected an error when no IDs are valid UUIDs") + } +} diff --git a/api/internal/searchclient/client.go b/api/internal/searchclient/client.go new file mode 100644 index 0000000..73c4a81 --- /dev/null +++ b/api/internal/searchclient/client.go @@ -0,0 +1,44 @@ +// Package searchclient adapts the generated gRPC SearchServiceClient to +// the narrow queryapi.searchClient interface, so queryapi doesn't need to +// know anything about gRPC/protobuf directly. +package searchclient + +import ( + "context" + "fmt" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + + searchv1 "github.com/sentry/sentry/proto/sentry/search/v1" +) + +type Client struct { + grpc searchv1.SearchServiceClient + conn *grpc.ClientConn +} + +// Dial connects to the search service. Plain TCP, no TLS: internal +// service-to-service traffic (api <-> search), same trust boundary as +// api's existing plain-TCP connection to ClickHouse -- mTLS in this +// project is specifically the agent<->ingest edge boundary, not every +// internal hop. +func Dial(addr string) (*Client, error) { + conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return nil, fmt.Errorf("dialing search service at %s: %w", addr, err) + } + return &Client{grpc: searchv1.NewSearchServiceClient(conn), conn: conn}, nil +} + +func (c *Client) Close() error { + return c.conn.Close() +} + +func (c *Client) Search(ctx context.Context, query string, limit uint32) ([]string, error) { + resp, err := c.grpc.Search(ctx, &searchv1.SearchRequest{Query: query, Limit: limit}) + if err != nil { + return nil, err + } + return resp.GetRecordIds(), nil +} diff --git a/docker-compose.yml b/docker-compose.yml index ec124c9..24f1f65 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,12 +1,18 @@ -# Phase 0 stack: Redpanda -> ingest -> ClickHouse -> api -> web. +# Phase 0+1 stack: Redpanda -> ingest -> ClickHouse -> api -> web, plus +# search (Tantivy full-text indexing, reads the same Redpanda topic +# ingest's consumer does). # # Does NOT include the Rust agent — see /agent/README.md: journald # sourcing needs the host's journal, which isn't something a container # gets for free. Run the agent natively on the host per # /docs/phase-0-runbook.md, pointed at ingest's mapped port (localhost:4317). +# Windows Event Log/ETW sourcing needs a real Windows host regardless — +# see /docs/phase-1-runbook.md. # # Before first run: generate dev mTLS certs (hack/dev-certs/generate.sh). -# See /docs/phase-0-runbook.md for the full sequence. +# See /docs/phase-0-runbook.md (Linux pipeline) and +# /docs/phase-1-runbook.md (Windows + full-text search) for the full +# sequences. services: redpanda: image: docker.redpanda.com/redpandadata/redpanda:v24.2.7 @@ -44,6 +50,10 @@ services: environment: REDPANDA_BROKERS: "redpanda:9092" REDPANDA_ADMIN_HOSTS: "redpanda:9644" + # Explicit rather than relying on both this script's and /search's + # defaults happening to agree — search consumes this same topic and + # needs to know the partition count up front (see /search/README.md). + REDPANDA_TOPIC_PARTITIONS: "6" clickhouse: image: clickhouse/clickhouse-server:24.8 @@ -109,9 +119,30 @@ services: volumes: - ./hack/dev-certs/out:/etc/sentry-ingest:ro + # Reads the same sentry.logs.raw topic ingest's consumer does (own + # offset tracking, own failure domain — see /search/README.md) and + # builds a Tantivy full-text index over the message field. + search: + build: + context: . # needs both search/ and proto/ + dockerfile: search/Dockerfile + container_name: sentry-search + depends_on: + redpanda-provision: + condition: service_completed_successfully + environment: + REDPANDA_BROKERS: "redpanda:9092" + REDPANDA_TOPIC_PARTITIONS: "6" # must match redpanda-provision's above + # tracing-subscriber's default filter suppresses INFO without this + # -- found by actually checking `docker compose logs search` and + # seeing nothing, same silent-logging gap the agent had in Phase 0. + RUST_LOG: "info" + volumes: + - search-index-data:/var/lib/sentry-search + api: build: - context: . + context: . # needs both api/ and proto/ (gRPC client to search) dockerfile: api/Dockerfile container_name: sentry-api depends_on: @@ -122,6 +153,7 @@ services: environment: CLICKHOUSE_ADDR: "clickhouse:9000" CLICKHOUSE_PASSWORD: "sentry-dev-only" + SEARCH_GRPC_ADDR: "search:50052" web: build: @@ -141,3 +173,4 @@ services: volumes: redpanda-data: clickhouse-data: + search-index-data: diff --git a/docs/phase-0-runbook.md b/docs/phase-0-runbook.md index 2e0f631..04f0e31 100644 --- a/docs/phase-0-runbook.md +++ b/docs/phase-0-runbook.md @@ -5,13 +5,16 @@ ingest, and ClickHouse, to a browser table. This is the actual "done" criterion for Phase 0 — if this doesn't work, Phase 0 isn't done, regardless of what any individual component's tests say. -**This sequence has not been run end-to-end** in the environment that -built it (no Docker available there — see the caveats each component's -summary already flagged). Individual pieces are unit-tested and built -successfully in isolation; this document is the logical sequence to run -for real, not a report that it's been run. Expect to debug something on -first attempt, and treat the "Troubleshooting" section at the bottom as a -starting point, not an exhaustive list. +**Update:** this sequence has since been run for real, more than once, +against a live Docker install — not just written and trusted. Two real +bugs turned up doing that (ClickHouse's official image silently disabling +network access without a password set; `rpk`'s exact flag syntax) and got +fixed; see the git history around the "Fix two bugs found by actually +running the Phase 0 pipeline end-to-end" commit if you want the details. +The steps below reflect what was actually run, not just planned. The +"Troubleshooting" section below is still worth reading first if something +doesn't work — it's not an exhaustive list, but it does reflect real +failures encountered, not hypothetical ones. ## Prerequisites @@ -113,12 +116,15 @@ Reading the system journal generally needs root (or membership in the by distro, root is the reliable path for this runbook): ```sh -sudo ./target/release/sentry-agent +sudo RUST_LOG=info ./target/release/sentry-agent ``` -Leave it running in this terminal — you should see a `connected to ingest -service` log line. If you see a TLS or connection error instead, stop -here and check the Troubleshooting section before continuing. +`RUST_LOG=info` matters: `tracing_subscriber`'s default filter is +otherwise strict enough to suppress even the startup log line, and the +agent will look like it's silently doing nothing. Leave it running in +this terminal — you should see a `connected to ingest service` log line. +If you see a TLS or connection error instead, stop here and check the +Troubleshooting section before continuing. ## 6. Generate a test log line diff --git a/docs/phase-1-runbook.md b/docs/phase-1-runbook.md new file mode 100644 index 0000000..af20ee3 --- /dev/null +++ b/docs/phase-1-runbook.md @@ -0,0 +1,243 @@ +# Phase 1 runbook + +Extends `/docs/phase-0-runbook.md` with Windows log collection and +full-text search. Read that one first — this assumes the Phase 0 stack +(dev certs, `docker compose up`, backend sanity check) already works; +Phase 1 layers on top of it, doesn't replace it. + +## What's actually been verified vs. what needs real Windows + +Unlike Phase 0's original draft, most of this runbook reflects steps +actually run in this session against a live stack, not just planned: + +- **Verified for real:** the full Linux pipeline through search (agent → + ingest's `record_id` assignment → Redpanda → both consumers → + ClickHouse *and* Tantivy → both `/query` and `/search` → the same + `record_id` back from both). The `windows-fixture` generator sending + Windows-*shaped* data through the same pipeline and being correctly + queryable both ways, including `winevt.*` attributes and severity + mapping. +- **Not verified, and can't be from the environment this was built in:** + the actual Windows agent binary — `EvtSubscribe`, ETW session creation, + Windows service registration. No Windows toolchain was available + anywhere (confirmed: only the Linux target's std library installed, no + rustup, no way to even `cargo check --target x86_64-pc-windows-*`). + Part C below is the logical sequence to run on a real or virtualized + Windows host, not a report that it's been run. + +## Prerequisites (beyond Phase 0's) + +- A Windows host or VM (Windows 10/11 or Windows Server) for Part C. +- `mingw-w64` if cross-compiling the Windows build from Linux (optional — + building natively on Windows with `rustup target add + x86_64-pc-windows-msvc` works too and needs no extra setup on the Linux + side). +- Administrator access on the Windows host, for service registration and + (if you enable it) ETW. + +## Part A: full-text search (Linux-only, no Windows needed) + +Only needs what Phase 0's runbook already set up. + +### A1. Bring the stack up (if not already) + +```sh +docker compose up -d --build +``` + +Same as Phase 0, now also builds and starts `search` (Tantivy full-text +indexing). Confirm it's actually logging — same `RUST_LOG` gap the agent +has by default: + +```sh +docker compose logs search +``` + +You should see "search gRPC server listening" and rskafka connecting to +all of `sentry.logs.raw`'s partitions. If you see nothing at all, check +`RUST_LOG=info` is set on the `search` service in `docker-compose.yml`. + +### A2. Generate a log line and confirm both query paths agree + +Follow Phase 0's runbook to get the agent running and generate a test +line (steps 4–6 there — mTLS certs, build, run, `logger`). Then, instead +of just checking `/query`, check both: + +```sh +curl -X POST http://localhost:8080/query -H 'Content-Type: application/json' \ + -d '{"sql": "SELECT record_id, message FROM logs ORDER BY timestamp DESC LIMIT 1"}' + +curl -X POST http://localhost:8080/search -H 'Content-Type: application/json' \ + -d '{"query": ""}' +``` + +**The `record_id` in both responses should match.** That's the actual +Phase 1 exit criterion (`/CLAUDE.md`) for the Linux half: the same +record, reachable both ways. If `/search` returns nothing yet, give it a +few more seconds — Tantivy commits on a timer (`COMMIT_INTERVAL_MS`, +default 2s), so there's a small window where a record is in ClickHouse +but not yet searchable. + +### A3. Confirm from the web UI + +Open `http://localhost:3000` — there are now two pages, linked via the +top nav: **SQL Query** (unchanged from Phase 0) and **Full-Text Search** +(new). Run the same free-text term on the search page and confirm you +see the row. + +## Part B: Windows-shaped data without a Windows host + +Still no Windows needed — this tests the pipeline's handling of +Windows-*shaped* data, not the real Windows integration (see +`/hack/windows-fixture/README.md` for the exact distinction). + +```sh +cd hack/windows-fixture +go run . --count 5 +``` + +Then repeat A2's pattern: query for one of the synthetic events by +`attributes['winevt.event_id']` via `/query`, and by a distinctive word +from its message via `/search`. Both should return it, and `attributes` +should carry `winevt.event_id`/`winevt.provider`/`winevt.channel`/ +`winevt.computer`. + +## Part C: the real Windows agent (needs actual Windows) + +### C1. Build + +On the Windows host itself (simplest — avoids cross-compilation +entirely): + +```powershell +rustup target add x86_64-pc-windows-msvc +cd agent +cargo build --release --target x86_64-pc-windows-msvc --no-default-features --features windows-eventlog,etw +``` + +Or cross-compile from Linux, then copy the binary over: + +```sh +rustup target add x86_64-pc-windows-gnu +cargo build --release --target x86_64-pc-windows-gnu --no-default-features --features windows-eventlog,etw +``` + +`protoc` needs to be on `PATH` either way (used by `tonic-build` at +compile time), same requirement as the Linux build. + +### C2. Get mTLS certs onto the Windows host + +Copy `hack/dev-certs/out/{ca,client,client-key}.pem` from wherever you +ran `generate.sh` to `C:\ProgramData\SentryAgent\` on the Windows host +(create the directory first). Same dev-only certs Phase 0's Linux agent +uses — the CA doesn't care what platform the client is on, only that the +client cert was signed by it. + +### C3. Config + +Create `C:\ProgramData\SentryAgent\agent.toml`: + +```toml +[source] +kind = "eventlog" +channels = ["Application", "System", "Security"] + +[ingest] +endpoint = "https://:4317" +``` + +If Docker Compose runs on a different machine than the Windows host, +`ingest`'s server cert SAN needs to cover that hostname/IP too — see +`hack/dev-certs/generate.sh` and regenerate with an updated SAN if +needed (same note as Phase 0's runbook's troubleshooting section). + +### C4. Run it directly first, before installing as a service + +```powershell +$env:RUST_LOG="info" +.\sentry-agent.exe --config C:\ProgramData\SentryAgent\agent.toml +``` + +Confirms the Event Log source and mTLS connection work before adding the +Windows service layer on top — if something's wrong, it's much easier to +diagnose here than after wrapping it in a service. + +### C5. Generate a Windows Event Log entry and confirm it flows through + +From another PowerShell window (or Event Viewer): + +```powershell +eventcreate /T INFORMATION /ID 1 /L APPLICATION /SO "SentryTest" /D "phase1 windows verification line" +``` + +Then check both query paths, same pattern as A2. + +### C6. Install as a service + +```powershell +.\sentry-agent.exe install +sc.exe start SentryAgent +``` + +Verify it's running (`sc.exe query SentryAgent`) and generate another +test event to confirm it's still flowing through while running as a +service, not just in the foreground. **Known gap:** no console under the +SCM means `tracing`'s log output currently has nowhere to go — see +`/agent/README.md`'s "Running as a Windows service" section. If +something goes wrong here, you're debugging blind until that's +addressed; C4's foreground run is where to diagnose real problems. + +```powershell +sc.exe stop SentryAgent +.\sentry-agent.exe uninstall +``` + +### C7 (optional). ETW + +Only if you actually want it running — **read the privilege section in +`/agent/README.md` first.** ETW needs elevated privileges (an +administrator token or `SeSystemProfilePrivilege`), a real consideration +for a log-shipping agent, not a formality. Providers are configured by +GUID (`logman query providers ""` to look one up): + +```toml +[source] +kind = "etw" +providers = ["{22FB2CD6-0E7B-422B-A0C7-2FAD1FD0E716}"] +``` + +### C8 (informational). WEF + +No new steps — see `/agent/README.md`'s WEF section. The supported +pattern is running this same agent (Event Log source) on a Windows +Server already acting as a native Windows Event Collector, pointed at +the `ForwardedEvents` channel instead of the usual three. A true +agentless WS-Management receiver is explicitly not built in Phase 1. + +## Troubleshooting (Phase 1-specific) + +**`/search` returns nothing but `/query` finds the record.** +Check `COMMIT_INTERVAL_MS` hasn't elapsed yet (default 2s) — Tantivy +batches commits, same reasoning as ClickHouse batching inserts. If it's +been well over that and still nothing: `docker compose logs search` — +look for "skipping record with empty record_id" (would mean something +upstream isn't assigning IDs — shouldn't happen) or connection errors to +Redpanda. + +**Windows agent connects but Event Log entries never show up.** +Check the channel name is exactly right (`Application`/`System`/ +`Security`, case matters to the Windows API) and that the account +running the agent has read access to that log — `Security` specifically +often needs elevated rights beyond what `Application`/`System` need. + +**Windows build fails to find `protoc`.** +Same requirement as the Linux build — install `protoc` and ensure it's +on `PATH` before `cargo build`. On Windows, the official protoc release +zip plus adding its `bin/` to `PATH` is the simplest route. + +**Nothing in this section covers the problem.** +Genuinely possible — this is the least-tested part of the whole Phase 1 +build (see the caveat at the top). Check `/agent/README.md`'s Windows +sections for the specific module involved (`source/windows_eventlog.rs`, +`source/etw.rs`, `service.rs`) and their own "UNVERIFIED" comments for +what's most likely to need a real fix. diff --git a/hack/README.md b/hack/README.md index 78776a9..a9c89c1 100644 --- a/hack/README.md +++ b/hack/README.md @@ -14,3 +14,8 @@ monorepos (Kubernetes among them). - `dev-certs/` — generates a throwaway CA + server/client cert pair for local mTLS between the agent and ingest. See `/docs/phase-0-runbook.md` for when to run it. +- `windows-fixture/` — sends synthetic Windows Event Log-shaped records + directly to `ingest`, bypassing the real Windows agent. Tests whether + the pipeline handles Windows-shaped data; doesn't test the real + `EvtSubscribe`/ETW integration, which needs actual Windows. See + `/docs/phase-1-runbook.md`. diff --git a/hack/windows-fixture/README.md b/hack/windows-fixture/README.md new file mode 100644 index 0000000..8ff7434 --- /dev/null +++ b/hack/windows-fixture/README.md @@ -0,0 +1,52 @@ +# windows-fixture + +Sends synthetic Windows Event Log-shaped `PushBatchRequest`s directly to +`ingest`'s gRPC endpoint, bypassing the actual Windows agent entirely. + +## What this does and doesn't test + +**Tests:** can the pipeline (ingest → ClickHouse → search → api → web) +correctly handle Windows-*shaped* data — the `winevt.*` attributes, the +`record_id` join between SQL and full-text search, Windows severity +levels mapping onto the right column values? This is exactly what's +automatable without a Windows host, and it's genuinely exercised: five +realistic, well-known Windows events (failed/successful logon, a service +state change, an application crash, an unexpected reboot) with real +EventIDs and providers. + +**Does not test:** whether the real Windows agent's `EvtSubscribe`/ETW +integration actually works, whether Windows service registration +succeeds, whether ETW session creation/provider enabling works. Those are +fundamentally different questions — they need a real or virtualized +Windows host, and nothing here pretends otherwise. See +`/docs/phase-1-runbook.md` for exactly which is which. + +## Running + +Requires the docker-compose stack up (`ingest` reachable, dev certs +generated): + +```sh +cd hack/windows-fixture +go run . --count 5 +``` + +``` +sent 5 synthetic Windows-shaped records, ingest accepted 5 + [SEVERITY_WARN] An account failed to log on. (event_id=4625 provider=Microsoft-Windows-Security-Auditing) + ... +``` + +Then confirm both query paths see it: + +```sh +curl -s -X POST http://localhost:8080/query -H 'Content-Type: application/json' \ + -d '{"sql": "SELECT host, severity, message, attributes['"'"'winevt.event_id'"'"'] AS event_id FROM logs WHERE host = '"'"'WIN-FIXTURE-01'"'"' ORDER BY timestamp DESC"}' + +curl -s -X POST http://localhost:8080/search -H 'Content-Type: application/json' \ + -d '{"query": "notepad"}' +``` + +Flags: `--addr` (default `localhost:4317`), `--ca`/`--cert`/`--key` +(default to `../dev-certs/out/{ca,client,client-key}.pem`), `--count` +(default 5, cycles through the fixed event list if higher). diff --git a/hack/windows-fixture/go.mod b/hack/windows-fixture/go.mod new file mode 100644 index 0000000..4388c87 --- /dev/null +++ b/hack/windows-fixture/go.mod @@ -0,0 +1,18 @@ +module github.com/sentry/sentry/hack/windows-fixture + +go 1.25.0 + +replace github.com/sentry/sentry/proto => ../../proto + +require ( + github.com/sentry/sentry/proto v0.0.0-00010101000000-000000000000 + google.golang.org/grpc v1.83.0 +) + +require ( + golang.org/x/net v0.55.0 // indirect + golang.org/x/sys v0.45.0 // indirect + golang.org/x/text v0.37.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/protobuf v1.36.12 // indirect +) diff --git a/hack/windows-fixture/go.sum b/hack/windows-fixture/go.sum new file mode 100644 index 0000000..fcc6745 --- /dev/null +++ b/hack/windows-fixture/go.sum @@ -0,0 +1,38 @@ +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ= +google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= diff --git a/hack/windows-fixture/main.go b/hack/windows-fixture/main.go new file mode 100644 index 0000000..41facdf --- /dev/null +++ b/hack/windows-fixture/main.go @@ -0,0 +1,130 @@ +// Command windows-fixture sends synthetic Windows Event Log-shaped +// PushBatchRequests directly to ingest's gRPC endpoint, bypassing the +// actual Windows agent entirely. +// +// This tests one specific thing: can the pipeline (ingest -> ClickHouse +// -> search -> api -> web) correctly handle Windows-*shaped* data (the +// winevt.* attributes, the record_id join, severity mapping)? It does +// NOT test whether the real Windows agent's EvtSubscribe/ETW integration +// actually works -- that's a fundamentally different question that can +// only be answered on a real or virtualized Windows host. See +// /docs/phase-1-runbook.md for exactly which is which. +package main + +import ( + "context" + "crypto/tls" + "crypto/x509" + "flag" + "fmt" + "os" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + + logsv1 "github.com/sentry/sentry/proto/sentry/logs/v1" +) + +func main() { + addr := flag.String("addr", "localhost:4317", "ingest gRPC address") + caFile := flag.String("ca", "../dev-certs/out/ca.pem", "CA cert path") + certFile := flag.String("cert", "../dev-certs/out/client.pem", "client cert path") + keyFile := flag.String("key", "../dev-certs/out/client-key.pem", "client key path") + count := flag.Int("count", 5, "number of synthetic events to send") + flag.Parse() + + tlsConf, err := loadTLSConfig(*caFile, *certFile, *keyFile) + if err != nil { + fmt.Fprintln(os.Stderr, "loading TLS config:", err) + os.Exit(1) + } + + conn, err := grpc.NewClient(*addr, grpc.WithTransportCredentials(credentials.NewTLS(tlsConf))) + if err != nil { + fmt.Fprintln(os.Stderr, "dialing ingest:", err) + os.Exit(1) + } + defer conn.Close() + + client := logsv1.NewLogIngestClient(conn) + records := syntheticWindowsRecords(*count) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + resp, err := client.PushBatch(ctx, &logsv1.PushBatchRequest{ + BatchId: "windows-fixture", + Records: records, + }) + if err != nil { + fmt.Fprintln(os.Stderr, "PushBatch failed:", err) + os.Exit(1) + } + + fmt.Printf("sent %d synthetic Windows-shaped records, ingest accepted %d\n", len(records), resp.GetAccepted()) + for _, rec := range records { + fmt.Printf(" [%s] %s (event_id=%s provider=%s)\n", + rec.GetSeverity(), rec.GetMessage(), rec.GetAttributes()["winevt.event_id"], rec.GetAttributes()["winevt.provider"]) + } +} + +func loadTLSConfig(caFile, certFile, keyFile string) (*tls.Config, error) { + caPEM, err := os.ReadFile(caFile) + if err != nil { + return nil, fmt.Errorf("reading CA cert %s: %w", caFile, err) + } + caPool := x509.NewCertPool() + if !caPool.AppendCertsFromPEM(caPEM) { + return nil, fmt.Errorf("no valid certificates found in %s", caFile) + } + + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, fmt.Errorf("loading client cert/key: %w", err) + } + + return &tls.Config{ + RootCAs: caPool, + Certificates: []tls.Certificate{cert}, + }, nil +} + +// A handful of realistic, well-known Windows Event Log entries (real +// EventIDs/providers/channels), cycled through if --count exceeds the +// list length. +func syntheticWindowsRecords(count int) []*logsv1.LogRecord { + events := []struct { + eventID string + provider string + channel string + level logsv1.Severity + message string + }{ + {"4625", "Microsoft-Windows-Security-Auditing", "Security", logsv1.Severity_SEVERITY_WARN, "An account failed to log on."}, + {"7036", "Service Control Manager", "System", logsv1.Severity_SEVERITY_INFO, "The Windows Update service entered the running state."}, + {"1000", "Application Error", "Application", logsv1.Severity_SEVERITY_ERROR, "Faulting application name: notepad.exe"}, + {"4624", "Microsoft-Windows-Security-Auditing", "Security", logsv1.Severity_SEVERITY_INFO, "An account was successfully logged on."}, + {"41", "Microsoft-Windows-Kernel-Power", "System", logsv1.Severity_SEVERITY_FATAL, "The system has rebooted without cleanly shutting down first."}, + } + + records := make([]*logsv1.LogRecord, 0, count) + for i := 0; i < count; i++ { + e := events[i%len(events)] + records = append(records, &logsv1.LogRecord{ + TimestampUnixNano: time.Now().UnixNano(), + Host: "WIN-FIXTURE-01", + Service: "default", + Severity: e.level, + Message: e.message, + Attributes: map[string]string{ + "winevt.event_id": e.eventID, + "winevt.provider": e.provider, + "winevt.channel": e.channel, + "winevt.computer": "WIN-FIXTURE-01", + "winevt.record_number": fmt.Sprintf("%d", 100000+i), + }, + }) + } + return records +} diff --git a/ingest/README.md b/ingest/README.md index 0df09a1..fe34bf6 100644 --- a/ingest/README.md +++ b/ingest/README.md @@ -4,9 +4,17 @@ Go service sitting between the Rust agent and ClickHouse. Two halves in one binary, selected with `--mode`: - **server** — mTLS gRPC front end (`LogIngest.PushBatch`) that agents - connect to. Forwards each record, proto-encoded and unchanged, onto - Redpanda. Does no normalization — kept thin so agent-facing latency isn't - coupled to ClickHouse write performance. + connect to. Assigns each record a server-side `record_id` (a UUID, + overwriting whatever the agent sent — agents always send it empty) and + otherwise forwards records proto-encoded onto Redpanda unchanged. Still + kept thin — one field assignment, no real normalization — so agent- + facing latency isn't coupled to ClickHouse write performance. + `record_id` has to be assigned exactly once, here, rather than + independently by each downstream consumer: Phase 1's Tantivy indexer + and the ClickHouse writer both read the same Redpanda messages and need + to agree on the same ID for the same record to join search hits back to + rows — two consumers generating their own IDs would produce mismatched + ones for what's supposed to be the same record. - **consumer** — reads back off Redpanda, normalizes into the ClickHouse row shape (`internal/normalize`), and batch-writes via the native protocol driver. Commits Redpanda offsets only after a successful ClickHouse @@ -35,6 +43,10 @@ egress an agent has. See `/docs/architecture.md`. protocol, pure Go (no cgo). - **golang.org/x/sync/errgroup** — used in `cmd/ingest/main.go` to run the server and consumer halves concurrently and propagate the first error. +- **github.com/google/uuid** — was already in the dependency graph + transitively (via clickhouse-go); promoted to a direct dependency for + `record_id` generation in `internal/grpcserver`, so not a new addition + to the transitive tree. ## Configuration diff --git a/ingest/go.mod b/ingest/go.mod index a5fbcb5..f801337 100644 --- a/ingest/go.mod +++ b/ingest/go.mod @@ -6,6 +6,7 @@ replace github.com/sentry/sentry/proto => ../proto require ( github.com/ClickHouse/clickhouse-go/v2 v2.48.0 + github.com/google/uuid v1.6.0 github.com/segmentio/kafka-go v0.4.51 github.com/sentry/sentry/proto v0.0.0-00010101000000-000000000000 golang.org/x/sync v0.22.0 @@ -19,7 +20,6 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/go-faster/city v1.0.1 // indirect github.com/go-faster/errors v0.7.1 // indirect - github.com/google/uuid v1.6.0 // indirect github.com/klauspost/compress v1.19.1 // indirect github.com/paulmach/orb v0.13.0 // indirect github.com/pierrec/lz4/v4 v4.1.27 // indirect diff --git a/ingest/internal/clickhousewriter/writer.go b/ingest/internal/clickhousewriter/writer.go index 37a3d85..ac32637 100644 --- a/ingest/internal/clickhousewriter/writer.go +++ b/ingest/internal/clickhousewriter/writer.go @@ -41,14 +41,14 @@ func (w *Writer) Close() error { } func (w *Writer) WriteBatch(ctx context.Context, records []*logsv1.LogRecord) error { - batch, err := w.conn.PrepareBatch(ctx, "INSERT INTO logs (timestamp, host, service, severity, message, attributes)") + batch, err := w.conn.PrepareBatch(ctx, "INSERT INTO logs (timestamp, host, service, severity, message, attributes, record_id)") if err != nil { return fmt.Errorf("preparing batch: %w", err) } for _, rec := range records { row := normalize.ToRow(rec) - if err := batch.Append(row.Timestamp, row.Host, row.Service, row.Severity, row.Message, row.Attributes); err != nil { + if err := batch.Append(row.Timestamp, row.Host, row.Service, row.Severity, row.Message, row.Attributes, row.RecordID); err != nil { return fmt.Errorf("appending row to batch: %w", err) } } diff --git a/ingest/internal/grpcserver/server.go b/ingest/internal/grpcserver/server.go index cd23fae..85e04f9 100644 --- a/ingest/internal/grpcserver/server.go +++ b/ingest/internal/grpcserver/server.go @@ -1,7 +1,9 @@ // Package grpcserver implements the agent-facing side of ingest: an mTLS -// gRPC server accepting LogIngest.PushBatch calls, which it forwards -// unchanged (proto-encoded) onto Redpanda. Normalization into the -// ClickHouse row shape happens later, on the consumer side. +// gRPC server accepting LogIngest.PushBatch calls. It assigns each record +// a stable record_id (see the proto field comment for why this has to +// happen exactly once, here, rather than in either downstream consumer) +// and otherwise forwards records unchanged onto Redpanda — normalization +// into the ClickHouse row shape happens later, on the consumer side. package grpcserver import ( @@ -10,6 +12,7 @@ import ( "log/slog" "net" + "github.com/google/uuid" "github.com/segmentio/kafka-go" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -76,6 +79,15 @@ func (s *Server) PushBatch(ctx context.Context, req *logsv1.PushBatchRequest) (* msgs := make([]kafka.Message, 0, len(req.GetRecords())) for _, rec := range req.GetRecords() { + // Assigned here, once, before this record is produced to + // Redpanda: the ClickHouse-writer consumer and the Tantivy- + // indexer consumer (Phase 1) both read the same Redpanda + // messages and need to agree on the same ID for the same + // record. Overwrites anything the agent sent (it always sends + // empty, per the proto comment, but this is authoritative + // regardless). + rec.RecordId = uuid.NewString() + val, err := proto.Marshal(rec) if err != nil { return nil, status.Errorf(codes.InvalidArgument, "marshaling record: %v", err) diff --git a/ingest/internal/grpcserver/server_test.go b/ingest/internal/grpcserver/server_test.go new file mode 100644 index 0000000..538fe0a --- /dev/null +++ b/ingest/internal/grpcserver/server_test.go @@ -0,0 +1,126 @@ +package grpcserver + +import ( + "context" + "io" + "log/slog" + "sync" + "testing" + + "github.com/segmentio/kafka-go" + "google.golang.org/protobuf/proto" + + "github.com/sentry/sentry/ingest/internal/config" + logsv1 "github.com/sentry/sentry/proto/sentry/logs/v1" +) + +type fakeProducer struct { + mu sync.Mutex + written [][]kafka.Message + err error +} + +func (f *fakeProducer) WriteBatch(_ context.Context, msgs []kafka.Message) error { + f.mu.Lock() + defer f.mu.Unlock() + if f.err != nil { + return f.err + } + batch := make([]kafka.Message, len(msgs)) + copy(batch, msgs) + f.written = append(f.written, batch) + return nil +} + +func newTestServer(p batchProducer) *Server { + return New(slog.New(slog.NewTextHandler(io.Discard, nil)), config.GRPCConfig{}, config.TLSConfig{}, p) +} + +func TestPushBatchAssignsRecordID(t *testing.T) { + fp := &fakeProducer{} + s := newTestServer(fp) + + req := &logsv1.PushBatchRequest{ + BatchId: "b1", + Records: []*logsv1.LogRecord{ + {Host: "h1", Message: "one"}, + {Host: "h1", Message: "two"}, + }, + } + + resp, err := s.PushBatch(context.Background(), req) + if err != nil { + t.Fatalf("PushBatch() error = %v", err) + } + if resp.GetAccepted() != 2 { + t.Fatalf("Accepted = %d, want 2", resp.GetAccepted()) + } + + fp.mu.Lock() + defer fp.mu.Unlock() + if len(fp.written) != 1 || len(fp.written[0]) != 2 { + t.Fatalf("unexpected written batches: %+v", fp.written) + } + + seen := make(map[string]bool) + for _, m := range fp.written[0] { + var rec logsv1.LogRecord + if err := proto.Unmarshal(m.Value, &rec); err != nil { + t.Fatalf("unmarshaling produced message: %v", err) + } + if rec.GetRecordId() == "" { + t.Fatalf("record_id was not assigned for message %q", rec.GetMessage()) + } + if seen[rec.GetRecordId()] { + t.Fatalf("duplicate record_id %q across records in the same batch", rec.GetRecordId()) + } + seen[rec.GetRecordId()] = true + } +} + +func TestPushBatchOverwritesAgentSuppliedRecordID(t *testing.T) { + fp := &fakeProducer{} + s := newTestServer(fp) + + req := &logsv1.PushBatchRequest{ + Records: []*logsv1.LogRecord{ + {Host: "h1", Message: "one", RecordId: "agent-supplied-should-be-ignored"}, + }, + } + + if _, err := s.PushBatch(context.Background(), req); err != nil { + t.Fatalf("PushBatch() error = %v", err) + } + + fp.mu.Lock() + defer fp.mu.Unlock() + var rec logsv1.LogRecord + if err := proto.Unmarshal(fp.written[0][0].Value, &rec); err != nil { + t.Fatalf("unmarshaling produced message: %v", err) + } + if rec.GetRecordId() == "agent-supplied-should-be-ignored" { + t.Fatal("expected ingest to overwrite any agent-supplied record_id") + } + if rec.GetRecordId() == "" { + t.Fatal("expected a server-assigned record_id") + } +} + +func TestPushBatchEmptyRecordsIsANoOp(t *testing.T) { + fp := &fakeProducer{} + s := newTestServer(fp) + + resp, err := s.PushBatch(context.Background(), &logsv1.PushBatchRequest{}) + if err != nil { + t.Fatalf("PushBatch() error = %v", err) + } + if resp.GetAccepted() != 0 { + t.Fatalf("Accepted = %d, want 0", resp.GetAccepted()) + } + + fp.mu.Lock() + defer fp.mu.Unlock() + if len(fp.written) != 0 { + t.Fatalf("expected no batches written for an empty request, got %d", len(fp.written)) + } +} diff --git a/ingest/internal/normalize/normalize.go b/ingest/internal/normalize/normalize.go index 5c3b6e4..0e2a511 100644 --- a/ingest/internal/normalize/normalize.go +++ b/ingest/internal/normalize/normalize.go @@ -9,6 +9,8 @@ package normalize import ( "time" + "github.com/google/uuid" + logsv1 "github.com/sentry/sentry/proto/sentry/logs/v1" ) @@ -19,6 +21,7 @@ type Row struct { Severity string Message string Attributes map[string]string + RecordID uuid.UUID } func ToRow(rec *logsv1.LogRecord) Row { @@ -26,6 +29,12 @@ func ToRow(rec *logsv1.LogRecord) Row { if attrs == nil { attrs = map[string]string{} } + // grpcserver's PushBatch handler always assigns a valid UUID before a + // record reaches this point (see its doc comment for why), so a parse + // failure here would mean something upstream is bypassing that — + // fall back to the nil UUID rather than failing the whole row, same + // "never silently drop a record" spirit as the rest of this pipeline. + recordID, _ := uuid.Parse(rec.GetRecordId()) return Row{ Timestamp: time.Unix(0, rec.GetTimestampUnixNano()).UTC(), Host: rec.GetHost(), @@ -33,6 +42,7 @@ func ToRow(rec *logsv1.LogRecord) Row { Severity: severityText(rec.GetSeverity()), Message: rec.GetMessage(), Attributes: attrs, + RecordID: recordID, } } diff --git a/ingest/internal/normalize/normalize_test.go b/ingest/internal/normalize/normalize_test.go index b8816cd..4d9b718 100644 --- a/ingest/internal/normalize/normalize_test.go +++ b/ingest/internal/normalize/normalize_test.go @@ -4,10 +4,13 @@ import ( "testing" "time" + "github.com/google/uuid" + logsv1 "github.com/sentry/sentry/proto/sentry/logs/v1" ) func TestToRowMapsFieldsAndSeverity(t *testing.T) { + id := uuid.New() rec := &logsv1.LogRecord{ TimestampUnixNano: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC).UnixNano(), Host: "host-1", @@ -15,6 +18,7 @@ func TestToRowMapsFieldsAndSeverity(t *testing.T) { Severity: logsv1.Severity_SEVERITY_ERROR, Message: "boom", Attributes: map[string]string{"k": "v"}, + RecordId: id.String(), } row := ToRow(rec) @@ -31,6 +35,17 @@ func TestToRowMapsFieldsAndSeverity(t *testing.T) { if !row.Timestamp.Equal(time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)) { t.Fatalf("unexpected timestamp: %v", row.Timestamp) } + if row.RecordID != id { + t.Fatalf("RecordID = %v, want %v", row.RecordID, id) + } +} + +func TestToRowInvalidRecordIDFallsBackToNilUUID(t *testing.T) { + rec := &logsv1.LogRecord{Host: "h", Service: "s", Message: "m", RecordId: "not-a-uuid"} + row := ToRow(rec) + if row.RecordID != uuid.Nil { + t.Fatalf("expected nil UUID fallback for an invalid record_id, got %v", row.RecordID) + } } func TestToRowNilAttributesBecomesEmptyMap(t *testing.T) { diff --git a/proto/sentry/logs/v1/logs.pb.go b/proto/sentry/logs/v1/logs.pb.go index 30cd2bc..cd6fc76 100644 --- a/proto/sentry/logs/v1/logs.pb.go +++ b/proto/sentry/logs/v1/logs.pb.go @@ -103,8 +103,18 @@ type LogRecord struct { // requirement in CLAUDE.md. Message string `protobuf:"bytes,5,opt,name=message,proto3" json:"message,omitempty"` // Structured fields extracted by the agent's parser (e.g. RFC 5424 - // syslog header fields). Empty when the raw-passthrough fallback fires. - Attributes map[string]string `protobuf:"bytes,6,rep,name=attributes,proto3" json:"attributes,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + // syslog header fields), plus source-provided fields (e.g. Windows + // Event Log's winevt.event_id/winevt.provider/winevt.channel). Empty + // when the raw-passthrough fallback fires and the source added nothing. + Attributes map[string]string `protobuf:"bytes,6,rep,name=attributes,proto3" json:"attributes,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + // Stable per-record identifier, used to join Tantivy full-text search + // hits back to their ClickHouse row (Phase 1). Always empty as sent by + // the agent — ingest's PushBatch handler assigns this server-side, + // once, before producing to Redpanda, since both the ClickHouse-writer + // consumer and the Tantivy-indexer consumer read the same Redpanda + // messages and need to agree on the same ID for the same record. See + // /ingest/README.md. + RecordId string `protobuf:"bytes,7,opt,name=record_id,json=recordId,proto3" json:"record_id,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -181,6 +191,13 @@ func (x *LogRecord) GetAttributes() map[string]string { return nil } +func (x *LogRecord) GetRecordId() string { + if x != nil { + return x.RecordId + } + return "" +} + type PushBatchRequest struct { state protoimpl.MessageState `protogen:"open.v1"` // Agent-assigned identifier for dedup/idempotency on retry. Ingest may @@ -287,7 +304,7 @@ var File_sentry_logs_v1_logs_proto protoreflect.FileDescriptor const file_sentry_logs_v1_logs_proto_rawDesc = "" + "\n" + - "\x19sentry/logs/v1/logs.proto\x12\x0esentry.logs.v1\"\xc3\x02\n" + + "\x19sentry/logs/v1/logs.proto\x12\x0esentry.logs.v1\"\xe0\x02\n" + "\tLogRecord\x12.\n" + "\x13timestamp_unix_nano\x18\x01 \x01(\x03R\x11timestampUnixNano\x12\x12\n" + "\x04host\x18\x02 \x01(\tR\x04host\x12\x18\n" + @@ -296,7 +313,8 @@ const file_sentry_logs_v1_logs_proto_rawDesc = "" + "\amessage\x18\x05 \x01(\tR\amessage\x12I\n" + "\n" + "attributes\x18\x06 \x03(\v2).sentry.logs.v1.LogRecord.AttributesEntryR\n" + - "attributes\x1a=\n" + + "attributes\x12\x1b\n" + + "\trecord_id\x18\a \x01(\tR\brecordId\x1a=\n" + "\x0fAttributesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"b\n" + diff --git a/proto/sentry/logs/v1/logs.proto b/proto/sentry/logs/v1/logs.proto index 28ce4c2..42d1f88 100644 --- a/proto/sentry/logs/v1/logs.proto +++ b/proto/sentry/logs/v1/logs.proto @@ -47,8 +47,19 @@ message LogRecord { string message = 5; // Structured fields extracted by the agent's parser (e.g. RFC 5424 - // syslog header fields). Empty when the raw-passthrough fallback fires. + // syslog header fields), plus source-provided fields (e.g. Windows + // Event Log's winevt.event_id/winevt.provider/winevt.channel). Empty + // when the raw-passthrough fallback fires and the source added nothing. map attributes = 6; + + // Stable per-record identifier, used to join Tantivy full-text search + // hits back to their ClickHouse row (Phase 1). Always empty as sent by + // the agent — ingest's PushBatch handler assigns this server-side, + // once, before producing to Redpanda, since both the ClickHouse-writer + // consumer and the Tantivy-indexer consumer read the same Redpanda + // messages and need to agree on the same ID for the same record. See + // /ingest/README.md. + string record_id = 7; } message PushBatchRequest { diff --git a/proto/sentry/search/v1/search.pb.go b/proto/sentry/search/v1/search.pb.go new file mode 100644 index 0000000..dc9a1d8 --- /dev/null +++ b/proto/sentry/search/v1/search.pb.go @@ -0,0 +1,192 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v7.35.1 +// source: sentry/search/v1/search.proto + +package searchv1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type SearchRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Free-text query passed to Tantivy's query parser as-is. Supports + // phrase queries ("exact phrase") and wildcards (foo*) per Tantivy's + // own query syntax — see /search/README.md for exactly what that does + // and doesn't support in Phase 1. + Query string `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` + // Max results to return. 0 (unset) uses the service's own default. + Limit uint32 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SearchRequest) Reset() { + *x = SearchRequest{} + mi := &file_sentry_search_v1_search_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SearchRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SearchRequest) ProtoMessage() {} + +func (x *SearchRequest) ProtoReflect() protoreflect.Message { + mi := &file_sentry_search_v1_search_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SearchRequest.ProtoReflect.Descriptor instead. +func (*SearchRequest) Descriptor() ([]byte, []int) { + return file_sentry_search_v1_search_proto_rawDescGZIP(), []int{0} +} + +func (x *SearchRequest) GetQuery() string { + if x != nil { + return x.Query + } + return "" +} + +func (x *SearchRequest) GetLimit() uint32 { + if x != nil { + return x.Limit + } + return 0 +} + +type SearchResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // record_ids of matching logs, most-relevant first. Callers join these + // back against ClickHouse's `logs.record_id` column to get full rows — + // this service only ever returns IDs, never row data, so it stays a + // pure text index rather than a second copy of the row. + RecordIds []string `protobuf:"bytes,1,rep,name=record_ids,json=recordIds,proto3" json:"record_ids,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SearchResponse) Reset() { + *x = SearchResponse{} + mi := &file_sentry_search_v1_search_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SearchResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SearchResponse) ProtoMessage() {} + +func (x *SearchResponse) ProtoReflect() protoreflect.Message { + mi := &file_sentry_search_v1_search_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SearchResponse.ProtoReflect.Descriptor instead. +func (*SearchResponse) Descriptor() ([]byte, []int) { + return file_sentry_search_v1_search_proto_rawDescGZIP(), []int{1} +} + +func (x *SearchResponse) GetRecordIds() []string { + if x != nil { + return x.RecordIds + } + return nil +} + +var File_sentry_search_v1_search_proto protoreflect.FileDescriptor + +const file_sentry_search_v1_search_proto_rawDesc = "" + + "\n" + + "\x1dsentry/search/v1/search.proto\x12\x10sentry.search.v1\";\n" + + "\rSearchRequest\x12\x14\n" + + "\x05query\x18\x01 \x01(\tR\x05query\x12\x14\n" + + "\x05limit\x18\x02 \x01(\rR\x05limit\"/\n" + + "\x0eSearchResponse\x12\x1d\n" + + "\n" + + "record_ids\x18\x01 \x03(\tR\trecordIds2\\\n" + + "\rSearchService\x12K\n" + + "\x06Search\x12\x1f.sentry.search.v1.SearchRequest\x1a .sentry.search.v1.SearchResponseB:Z8github.com/sentry/sentry/proto/sentry/search/v1;searchv1b\x06proto3" + +var ( + file_sentry_search_v1_search_proto_rawDescOnce sync.Once + file_sentry_search_v1_search_proto_rawDescData []byte +) + +func file_sentry_search_v1_search_proto_rawDescGZIP() []byte { + file_sentry_search_v1_search_proto_rawDescOnce.Do(func() { + file_sentry_search_v1_search_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_sentry_search_v1_search_proto_rawDesc), len(file_sentry_search_v1_search_proto_rawDesc))) + }) + return file_sentry_search_v1_search_proto_rawDescData +} + +var file_sentry_search_v1_search_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_sentry_search_v1_search_proto_goTypes = []any{ + (*SearchRequest)(nil), // 0: sentry.search.v1.SearchRequest + (*SearchResponse)(nil), // 1: sentry.search.v1.SearchResponse +} +var file_sentry_search_v1_search_proto_depIdxs = []int32{ + 0, // 0: sentry.search.v1.SearchService.Search:input_type -> sentry.search.v1.SearchRequest + 1, // 1: sentry.search.v1.SearchService.Search:output_type -> sentry.search.v1.SearchResponse + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_sentry_search_v1_search_proto_init() } +func file_sentry_search_v1_search_proto_init() { + if File_sentry_search_v1_search_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_sentry_search_v1_search_proto_rawDesc), len(file_sentry_search_v1_search_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_sentry_search_v1_search_proto_goTypes, + DependencyIndexes: file_sentry_search_v1_search_proto_depIdxs, + MessageInfos: file_sentry_search_v1_search_proto_msgTypes, + }.Build() + File_sentry_search_v1_search_proto = out.File + file_sentry_search_v1_search_proto_goTypes = nil + file_sentry_search_v1_search_proto_depIdxs = nil +} diff --git a/proto/sentry/search/v1/search.proto b/proto/sentry/search/v1/search.proto new file mode 100644 index 0000000..2a789a5 --- /dev/null +++ b/proto/sentry/search/v1/search.proto @@ -0,0 +1,33 @@ +syntax = "proto3"; + +package sentry.search.v1; + +option go_package = "github.com/sentry/sentry/proto/sentry/search/v1;searchv1"; + +// SearchService is the full-text search index (Tantivy-backed) that +// `api` calls to resolve a free-text query into matching record_ids, +// which `api` then joins back against ClickHouse. Internal service-to- +// service call, same gRPC-first convention as agent<->ingest — see +// /docs/architecture.md and /search/README.md. +service SearchService { + rpc Search(SearchRequest) returns (SearchResponse); +} + +message SearchRequest { + // Free-text query passed to Tantivy's query parser as-is. Supports + // phrase queries ("exact phrase") and wildcards (foo*) per Tantivy's + // own query syntax — see /search/README.md for exactly what that does + // and doesn't support in Phase 1. + string query = 1; + + // Max results to return. 0 (unset) uses the service's own default. + uint32 limit = 2; +} + +message SearchResponse { + // record_ids of matching logs, most-relevant first. Callers join these + // back against ClickHouse's `logs.record_id` column to get full rows — + // this service only ever returns IDs, never row data, so it stays a + // pure text index rather than a second copy of the row. + repeated string record_ids = 1; +} diff --git a/proto/sentry/search/v1/search_grpc.pb.go b/proto/sentry/search/v1/search_grpc.pb.go new file mode 100644 index 0000000..9eab644 --- /dev/null +++ b/proto/sentry/search/v1/search_grpc.pb.go @@ -0,0 +1,133 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc v7.35.1 +// source: sentry/search/v1/search.proto + +package searchv1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + SearchService_Search_FullMethodName = "/sentry.search.v1.SearchService/Search" +) + +// SearchServiceClient is the client API for SearchService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// SearchService is the full-text search index (Tantivy-backed) that +// `api` calls to resolve a free-text query into matching record_ids, +// which `api` then joins back against ClickHouse. Internal service-to- +// service call, same gRPC-first convention as agent<->ingest — see +// /docs/architecture.md and /search/README.md. +type SearchServiceClient interface { + Search(ctx context.Context, in *SearchRequest, opts ...grpc.CallOption) (*SearchResponse, error) +} + +type searchServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewSearchServiceClient(cc grpc.ClientConnInterface) SearchServiceClient { + return &searchServiceClient{cc} +} + +func (c *searchServiceClient) Search(ctx context.Context, in *SearchRequest, opts ...grpc.CallOption) (*SearchResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(SearchResponse) + err := c.cc.Invoke(ctx, SearchService_Search_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// SearchServiceServer is the server API for SearchService service. +// All implementations must embed UnimplementedSearchServiceServer +// for forward compatibility. +// +// SearchService is the full-text search index (Tantivy-backed) that +// `api` calls to resolve a free-text query into matching record_ids, +// which `api` then joins back against ClickHouse. Internal service-to- +// service call, same gRPC-first convention as agent<->ingest — see +// /docs/architecture.md and /search/README.md. +type SearchServiceServer interface { + Search(context.Context, *SearchRequest) (*SearchResponse, error) + mustEmbedUnimplementedSearchServiceServer() +} + +// UnimplementedSearchServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedSearchServiceServer struct{} + +func (UnimplementedSearchServiceServer) Search(context.Context, *SearchRequest) (*SearchResponse, error) { + return nil, status.Error(codes.Unimplemented, "method Search not implemented") +} +func (UnimplementedSearchServiceServer) mustEmbedUnimplementedSearchServiceServer() {} +func (UnimplementedSearchServiceServer) testEmbeddedByValue() {} + +// UnsafeSearchServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to SearchServiceServer will +// result in compilation errors. +type UnsafeSearchServiceServer interface { + mustEmbedUnimplementedSearchServiceServer() +} + +func RegisterSearchServiceServer(s grpc.ServiceRegistrar, srv SearchServiceServer) { + // If the following call panics, it indicates UnimplementedSearchServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&SearchService_ServiceDesc, srv) +} + +func _SearchService_Search_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SearchRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SearchServiceServer).Search(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: SearchService_Search_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SearchServiceServer).Search(ctx, req.(*SearchRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// SearchService_ServiceDesc is the grpc.ServiceDesc for SearchService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var SearchService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "sentry.search.v1.SearchService", + HandlerType: (*SearchServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Search", + Handler: _SearchService_Search_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "sentry/search/v1/search.proto", +} diff --git a/search/Cargo.lock b/search/Cargo.lock new file mode 100644 index 0000000..e932021 --- /dev/null +++ b/search/Cargo.lock @@ -0,0 +1,2354 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower 0.5.3", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bitpacking" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96a7139abd3d9cebf8cd6f920a389cf3dc9576172e32f4563f188cae3c3eb019" +dependencies = [ + "crunchy", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "census" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "num-traits", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "serde_core", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastdivide" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "fs4" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8" +dependencies = [ + "rustix 0.38.44", + "windows-sys 0.52.0", +] + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap 2.14.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "htmlescape" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "libc", + "pin-project-lite", + "socket2 0.6.5", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "integer-encoding" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c00403deb17c3221a1fe4fb571b9ed0370b3dcd116553c77fa294a3d918699" + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "levenshtein_automata" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "lz4" +version = "1.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" +dependencies = [ + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "lz4_flex" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "measure_time" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbefd235b0aadd181626f281e1d684e116972988c14c264e42069d5e8a5775cc" +dependencies = [ + "instant", + "log", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "murmurhash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oneshot" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" + +[[package]] +name = "ownedbytes" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3a059efb063b8f425b948e042e6b9bd85edfe60e913630ed727b23e2dfcc558" +dependencies = [ + "stable_deref_trait", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap 2.14.0", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools 0.14.0", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rsasl" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed828a88913fd477c73bc3768b05d4b335ee775e29f2397bb59b9a4dd69ffb83" +dependencies = [ + "base64", + "digest", + "hmac", + "pbkdf2", + "rand", + "serde", + "serde_json", + "sha2", + "stringprep", + "thiserror 2.0.20", +] + +[[package]] +name = "rskafka" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "849b87417a191e37e16b8893eba8928bbd10e1989e6b57c4f8531549eaa29bdc" +dependencies = [ + "bytes", + "chrono", + "crc32c", + "flate2", + "futures", + "integer-encoding", + "lz4", + "parking_lot", + "rand", + "rsasl", + "snap", + "thiserror 1.0.69", + "tokio", + "tracing", + "zstd", +] + +[[package]] +name = "rust-stemmers" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" +dependencies = [ + "serde", + "serde_derive", +] + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.12.1", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "sentry-search" +version = "0.1.0" +dependencies = [ + "anyhow", + "prost", + "rskafka", + "serde", + "serde_json", + "tantivy", + "tempfile", + "tokio", + "tonic", + "tonic-build", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "sketches-ddsketch" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c" +dependencies = [ + "serde", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "snap" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886" + +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + +[[package]] +name = "tantivy" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96599ea6fccd844fc833fed21d2eecac2e6a7c1afd9e044057391d78b1feb141" +dependencies = [ + "aho-corasick", + "arc-swap", + "base64", + "bitpacking", + "byteorder", + "census", + "crc32fast", + "crossbeam-channel", + "downcast-rs", + "fastdivide", + "fnv", + "fs4", + "htmlescape", + "itertools 0.12.1", + "levenshtein_automata", + "log", + "lru", + "lz4_flex", + "measure_time", + "memmap2", + "num_cpus", + "once_cell", + "oneshot", + "rayon", + "regex", + "rust-stemmers", + "rustc-hash", + "serde", + "serde_json", + "sketches-ddsketch", + "smallvec", + "tantivy-bitpacker", + "tantivy-columnar", + "tantivy-common", + "tantivy-fst", + "tantivy-query-grammar", + "tantivy-stacker", + "tantivy-tokenizer-api", + "tempfile", + "thiserror 1.0.69", + "time", + "uuid", + "winapi", +] + +[[package]] +name = "tantivy-bitpacker" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284899c2325d6832203ac6ff5891b297fc5239c3dc754c5bc1977855b23c10df" +dependencies = [ + "bitpacking", +] + +[[package]] +name = "tantivy-columnar" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12722224ffbe346c7fec3275c699e508fd0d4710e629e933d5736ec524a1f44e" +dependencies = [ + "downcast-rs", + "fastdivide", + "itertools 0.12.1", + "serde", + "tantivy-bitpacker", + "tantivy-common", + "tantivy-sstable", + "tantivy-stacker", +] + +[[package]] +name = "tantivy-common" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8019e3cabcfd20a1380b491e13ff42f57bb38bf97c3d5fa5c07e50816e0621f4" +dependencies = [ + "async-trait", + "byteorder", + "ownedbytes", + "serde", + "time", +] + +[[package]] +name = "tantivy-fst" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18" +dependencies = [ + "byteorder", + "regex-syntax", + "utf8-ranges", +] + +[[package]] +name = "tantivy-query-grammar" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "847434d4af57b32e309f4ab1b4f1707a6c566656264caa427ff4285c4d9d0b82" +dependencies = [ + "nom", +] + +[[package]] +name = "tantivy-sstable" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c69578242e8e9fc989119f522ba5b49a38ac20f576fc778035b96cc94f41f98e" +dependencies = [ + "tantivy-bitpacker", + "tantivy-common", + "tantivy-fst", + "zstd", +] + +[[package]] +name = "tantivy-stacker" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56d6ff5591fc332739b3ce7035b57995a3ce29a93ffd6012660e0949c956ea8" +dependencies = [ + "murmurhash32", + "rand_distr", + "tantivy-common", +] + +[[package]] +name = "tantivy-tokenizer-api" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0dcade25819a89cfe6f17d932c9cedff11989936bf6dd4f336d50392053b04" +dependencies = [ + "serde", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix 1.1.4", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl 2.0.20", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2 0.6.5", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tonic" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "socket2 0.5.10", + "tokio", + "tokio-stream", + "tower 0.4.13", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9557ce109ea773b399c9b9e5dca39294110b74f1f342cb347a80d1fce8c26a11" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + +[[package]] +name = "utf8-ranges" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" + +[[package]] +name = "uuid" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/search/Cargo.toml b/search/Cargo.toml new file mode 100644 index 0000000..b9a711a --- /dev/null +++ b/search/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "sentry-search" +version = "0.1.0" +edition = "2021" +license = "AGPL-3.0-only" +description = "Sentry Tantivy-backed full-text search service" + +[[bin]] +name = "sentry-search" +path = "src/main.rs" + +[dependencies] +tokio = { version = "1", features = ["rt-multi-thread", "macros", "fs", "sync", "signal", "time"] } +tonic = "0.12" +prost = "0.13" + +tantivy = "0.22" +rskafka = "0.6" + +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +anyhow = "1" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } + +[build-dependencies] +tonic-build = "0.12" + +[dev-dependencies] +# Already in the dependency graph transitively (tantivy/tonic-build both +# pull it in); promoted to a direct dev-dependency for test use. +tempfile = "3" diff --git a/search/Dockerfile b/search/Dockerfile new file mode 100644 index 0000000..0d45b8a --- /dev/null +++ b/search/Dockerfile @@ -0,0 +1,20 @@ +# Build context must be the repo root (sentry/), since this needs both +# search/ and proto/: +# docker build -f search/Dockerfile -t sentry-search . +# +# Unlike /agent, this doesn't need musl/static linking -- it's a normal +# server-side container, not an edge-deployed binary, and both Tantivy and +# rskafka are pure Rust (no C deps to link against). distroless/cc rather +# than distroless/static: a normal dynamically-linked binary, not a fully +# static one. +FROM rust:1-slim AS builder +RUN apt-get update && apt-get install -y --no-install-recommends protobuf-compiler && rm -rf /var/lib/apt/lists/* +WORKDIR /src +COPY proto ./proto +COPY search ./search +WORKDIR /src/search +RUN cargo build --release + +FROM gcr.io/distroless/cc-debian12 +COPY --from=builder /src/search/target/release/sentry-search /sentry-search +ENTRYPOINT ["/sentry-search"] diff --git a/search/README.md b/search/README.md new file mode 100644 index 0000000..0315142 --- /dev/null +++ b/search/README.md @@ -0,0 +1,100 @@ +# search + +Tantivy-backed full-text search over log messages. Phase 1's answer to +"grep across everything," separate from ClickHouse's structured/ +aggregation queries. + +## Why a separate service, not embedded in ingest + +Tantivy is a Rust library with no maintained Go bindings — using it from +`ingest` (Go) would mean cgo-bridging to a compiled Rust cdylib, exactly +the fragile FFI complexity CLAUDE.md's "prefer boring, well-understood +dependencies... operators need to trust it" principle steers away from. +It would also couple ClickHouse-write latency to Tantivy-write latency in +the same request path. See `/docs/architecture.md` for the fuller +tradeoff writeup (dual-write vs. second-consumer-group) from Phase 1 +planning. + +## How it fits together + +``` +ingest (gRPC front end) --> Redpanda (sentry.logs.raw) --> ingest's ClickHouse-writer consumer --> ClickHouse + \ + `--> search's own consumer --> Tantivy index +``` + +`search` reads the *same* Redpanda topic `ingest`'s ClickHouse-writer +consumer reads, as an independent consumer in spirit (own offset +tracking, own failure domain — see "Offset tracking" below) even though +it's a completely separate process/service. If Tantivy indexing lags or +crashes, ClickHouse ingestion is completely unaffected. + +`api` calls `search`'s `SearchService.Search` gRPC RPC (see +`/proto/sentry/search/v1/search.proto`) to resolve a free-text query into +matching `record_id`s, then joins those back against ClickHouse's +`logs.record_id` column (see `/storage/migrations/0002_add_record_id.sql`) +to get full rows. `search` only ever returns IDs, never row data — it +stays a pure text index, not a second copy of the row. + +## Offset tracking: why this isn't a Kafka consumer group + +`rskafka` (chosen for being pure Rust, no cgo — consistent with why +`ingest` chose `segmentio/kafka-go` over `confluent-kafka-go`) is a +low-level client: it doesn't implement Kafka's broker-side consumer-group +coordination protocol the way `kafka-go` or `librdkafka` do. So `search` +tracks its own per-partition offsets in a plain JSON file next to the +Tantivy index (`offsets.rs`), persisted after each fetched batch. + +This is deliberately best-effort, not exactly-once: if the process dies +between processing a record and persisting its offset, that record gets +reprocessed after restart. This is safe because `SearchIndex::upsert` is +**delete-then-add on `record_id`** — Tantivy segments are immutable, so +this is the standard idiom for updates anyway, and it happens to make +reprocessing idempotent for free. Partition count is read from +`REDPANDA_TOPIC_PARTITIONS` (must match what +`/transport/provision-topics.sh` actually created — same kind of +documented cross-component contract as the topic name itself), not +discovered dynamically. + +## Query syntax + +Whatever Tantivy's own `QueryParser` supports against the `message` +field: plain terms, `"exact phrase"` queries, and `foo*` wildcards. Not +documented further here because it's Tantivy's syntax, not Sentry's — see +[Tantivy's query parser docs](https://docs.rs/tantivy/latest/tantivy/query/struct.QueryParser.html) +for the full grammar. No unified query language yet; that's Phase 2. + +## Configuration + +Environment variables (see `src/config.rs`): + +| Var | Default | Purpose | +|---|---|---| +| `GRPC_LISTEN_ADDR` | `0.0.0.0:50052` | Full socket address — Rust's parser needs one, unlike Go's `:PORT` shorthand `ingest`/`api` use | +| `REDPANDA_BROKERS` | `localhost:9092` | Comma-separated broker list | +| `REDPANDA_TOPIC` | `sentry.logs.raw` | Must match `/ingest`'s topic | +| `REDPANDA_TOPIC_PARTITIONS` | `6` | Must match what `/transport/provision-topics.sh` created | +| `INDEX_PATH` | `/var/lib/sentry-search/index` | Tantivy index directory | +| `OFFSETS_PATH` | `/var/lib/sentry-search/offsets.json` | Offset tracking file | +| `COMMIT_INTERVAL_MS` | `2000` | How often buffered writes become searchable | + +## Building & testing + +```sh +cargo build --release +cargo clippy --all-targets -- -D warnings +cargo test +``` + +`index.rs`'s tests run against a real (temp-directory) Tantivy index — +no external service needed, unlike ClickHouse. They cover the +delete-then-add idempotency, phrase queries, result limits, and the +before-commit/after-commit visibility boundary. `consumer.rs` (the +rskafka wiring) is not unit-tested — that needs a real Redpanda, same +category of gap as `/ingest`'s `kafka.Reader`/`kafka.Writer` wiring, and +is exercised by the docker-compose end-to-end flow instead. + +```sh +# from the repo root, not search/ +docker build -f search/Dockerfile -t sentry-search . +``` diff --git a/search/build.rs b/search/build.rs new file mode 100644 index 0000000..908c3f2 --- /dev/null +++ b/search/build.rs @@ -0,0 +1,14 @@ +fn main() -> Result<(), Box> { + // logs.proto is compiled only for the LogRecord message type (to + // decode what's read off Redpanda) -- this service never calls or + // implements LogIngest. search.proto is compiled for the + // SearchService server this binary implements. + tonic_build::configure().compile_protos( + &[ + "../proto/sentry/logs/v1/logs.proto", + "../proto/sentry/search/v1/search.proto", + ], + &["../proto"], + )?; + Ok(()) +} diff --git a/search/src/config.rs b/search/src/config.rs new file mode 100644 index 0000000..27b3542 --- /dev/null +++ b/search/src/config.rs @@ -0,0 +1,44 @@ +use anyhow::{Context, Result}; +use std::path::PathBuf; +use std::time::Duration; + +/// All via environment variables, same convention as /ingest and /api — +/// no config file format for this service either. +pub struct Config { + pub grpc_listen_addr: String, + pub redpanda_brokers: Vec, + pub redpanda_topic: String, + pub index_path: PathBuf, + pub offsets_path: PathBuf, + pub commit_interval: Duration, +} + +impl Config { + pub fn load() -> Result { + let commit_interval_ms: u64 = getenv("COMMIT_INTERVAL_MS", "2000") + .parse() + .context("COMMIT_INTERVAL_MS must be a number")?; + + Ok(Self { + // Rust's SocketAddr parser needs a full address, unlike Go's + // net package (ingest/api's ":PORT" convention won't parse + // here). + grpc_listen_addr: getenv("GRPC_LISTEN_ADDR", "0.0.0.0:50052"), + redpanda_brokers: getenv("REDPANDA_BROKERS", "localhost:9092") + .split(',') + .map(str::to_string) + .collect(), + redpanda_topic: getenv("REDPANDA_TOPIC", "sentry.logs.raw"), + index_path: PathBuf::from(getenv("INDEX_PATH", "/var/lib/sentry-search/index")), + offsets_path: PathBuf::from(getenv( + "OFFSETS_PATH", + "/var/lib/sentry-search/offsets.json", + )), + commit_interval: Duration::from_millis(commit_interval_ms), + }) + } +} + +fn getenv(key: &str, fallback: &str) -> String { + std::env::var(key).unwrap_or_else(|_| fallback.to_string()) +} diff --git a/search/src/consumer.rs b/search/src/consumer.rs new file mode 100644 index 0000000..8c45599 --- /dev/null +++ b/search/src/consumer.rs @@ -0,0 +1,134 @@ +use anyhow::{Context, Result}; +use prost::Message; +use rskafka::client::partition::UnknownTopicHandling; +use rskafka::client::ClientBuilder; +use std::sync::Arc; +use tokio::sync::Mutex; + +use crate::config::Config; +use crate::index::SearchIndex; +use crate::logsv1; +use crate::offsets::OffsetStore; + +/// Reads the same `sentry.logs.raw` topic ingest's ClickHouse-writer +/// consumer reads, as an independent consumer group in spirit (its own +/// offset tracking, own failure domain) even though rskafka doesn't speak +/// Kafka's broker-side consumer-group protocol -- see offsets.rs. One +/// task per partition; partition count comes from config rather than +/// discovered dynamically, since it has to match what +/// /transport/provision-topics.sh actually created anyway (documented +/// cross-component contract, same as the topic name already is). +pub async fn run(cfg: Arc, index: Arc, partition_count: i32) -> Result<()> { + let client = ClientBuilder::new(cfg.redpanda_brokers.clone()) + .build() + .await + .context("building rskafka client")?; + let client = Arc::new(client); + + let offsets = OffsetStore::load(&cfg.offsets_path) + .await + .context("loading offset store")?; + let offsets = Arc::new(Mutex::new(offsets)); + + // Periodic Tantivy commit, batched for throughput the same way + // ingest's ClickHouse writer batches inserts rather than inserting + // per-record. + let commit_interval = cfg.commit_interval; + let index_for_commit = Arc::clone(&index); + tokio::spawn(async move { + let mut ticker = tokio::time::interval(commit_interval); + loop { + ticker.tick().await; + if let Err(e) = index_for_commit.commit().await { + tracing::error!(error = %e, "periodic tantivy commit failed"); + } + } + }); + + let mut handles = Vec::with_capacity(partition_count as usize); + for partition in 0..partition_count { + let start_offset = offsets.lock().await.get(partition); + let client = Arc::clone(&client); + let index = Arc::clone(&index); + let offsets = Arc::clone(&offsets); + let topic = cfg.redpanda_topic.clone(); + handles.push(tokio::spawn(async move { + consume_partition(client, topic, partition, start_offset, index, offsets).await + })); + } + + for handle in handles { + handle + .await + .context("partition consumer task panicked")??; + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +async fn consume_partition( + client: Arc, + topic: String, + partition: i32, + start_offset: i64, + index: Arc, + offsets: Arc>, +) -> Result<()> { + let partition_client = client + .partition_client(topic.clone(), partition, UnknownTopicHandling::Error) + .await + .with_context(|| format!("creating partition client for {topic}[{partition}]"))?; + + let mut offset = start_offset; + loop { + let (records, _high_watermark) = partition_client + .fetch_records(offset, 1..1_000_000, 5_000) + .await + .with_context(|| { + format!("fetching records from {topic}[{partition}] at offset {offset}") + })?; + + if records.is_empty() { + continue; + } + + for record_and_offset in &records { + offset = record_and_offset.offset + 1; + + let Some(value) = &record_and_offset.record.value else { + continue; + }; + let rec = match logsv1::LogRecord::decode(value.as_slice()) { + Ok(rec) => rec, + Err(e) => { + tracing::warn!(error = %e, partition, "skipping unparseable message"); + continue; + } + }; + + if rec.record_id.is_empty() { + // Shouldn't happen -- ingest's gRPC front end always + // assigns this before producing -- but a message with no + // ID can't be joined back to a ClickHouse row, so it's + // useless to index. + tracing::warn!(partition, "skipping record with empty record_id"); + continue; + } + + if let Err(e) = index.upsert(&rec.record_id, &rec.message).await { + tracing::error!(error = %e, record_id = %rec.record_id, "failed to index record"); + } + } + + // Persisted after each fetched batch, not per-record: worst-case + // reprocessing on an unclean restart is one batch, which + // `SearchIndex::upsert`'s delete-then-add makes harmless anyway. + { + let mut offsets = offsets.lock().await; + offsets.set(partition, offset); + if let Err(e) = offsets.persist().await { + tracing::error!(error = %e, partition, "failed to persist offset"); + } + } + } +} diff --git a/search/src/grpc.rs b/search/src/grpc.rs new file mode 100644 index 0000000..c47e197 --- /dev/null +++ b/search/src/grpc.rs @@ -0,0 +1,48 @@ +use std::sync::Arc; +use tonic::{Request, Response, Status}; + +use crate::index::SearchIndex; +use crate::searchv1; + +const DEFAULT_LIMIT: usize = 100; + +pub struct SearchServer { + index: Arc, +} + +impl SearchServer { + pub fn new(index: Arc) -> Self { + Self { index } + } +} + +#[tonic::async_trait] +impl searchv1::search_service_server::SearchService for SearchServer { + async fn search( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + if req.query.trim().is_empty() { + return Err(Status::invalid_argument("query must not be empty")); + } + + let limit = if req.limit == 0 { + DEFAULT_LIMIT + } else { + req.limit as usize + }; + + let index = Arc::clone(&self.index); + let query = req.query.clone(); + // Tantivy's searcher is synchronous; run it on a blocking thread + // so it doesn't stall the async runtime alongside the consumer + // tasks. + let record_ids = tokio::task::spawn_blocking(move || index.search(&query, limit)) + .await + .map_err(|e| Status::internal(format!("search task panicked: {e}")))? + .map_err(|e| Status::invalid_argument(format!("search failed: {e}")))?; + + Ok(Response::new(searchv1::SearchResponse { record_ids })) + } +} diff --git a/search/src/index.rs b/search/src/index.rs new file mode 100644 index 0000000..a443125 --- /dev/null +++ b/search/src/index.rs @@ -0,0 +1,192 @@ +use anyhow::{Context, Result}; +use std::path::Path; +use tantivy::collector::TopDocs; +use tantivy::query::QueryParser; +use tantivy::schema::{Schema, Value, STORED, STRING, TEXT}; +use tantivy::{doc, Index, IndexReader, IndexWriter, ReloadPolicy, TantivyDocument, Term}; +use tokio::sync::Mutex; + +/// Minimal Tantivy index: a stable `record_id` (stored, exact-match) and +/// tokenized `message` text. Everything else (timestamp, host, service, +/// severity) is fetched by joining `record_id` back against ClickHouse in +/// `/api`'s search handler, not duplicated in here — this stays a pure +/// text index, not a second copy of the row. +pub struct SearchIndex { + index: Index, + writer: Mutex, + reader: IndexReader, + record_id_field: tantivy::schema::Field, + message_field: tantivy::schema::Field, +} + +/// 50MB is Tantivy's own suggested minimum writer heap budget; Phase 1 +/// has no real sizing data yet to tune this against. +const WRITER_HEAP_BYTES: usize = 50_000_000; + +impl SearchIndex { + pub fn open_or_create(path: &Path) -> Result { + std::fs::create_dir_all(path).context("creating tantivy index directory")?; + + let mut schema_builder = Schema::builder(); + let record_id_field = schema_builder.add_text_field("record_id", STRING | STORED); + let message_field = schema_builder.add_text_field("message", TEXT); + let schema = schema_builder.build(); + + let dir = tantivy::directory::MmapDirectory::open(path) + .context("opening tantivy mmap directory")?; + let index = + Index::open_or_create(dir, schema).context("opening/creating tantivy index")?; + + let writer = index + .writer(WRITER_HEAP_BYTES) + .context("creating tantivy index writer")?; + + let reader = index + .reader_builder() + .reload_policy(ReloadPolicy::OnCommitWithDelay) + .try_into() + .context("building tantivy index reader")?; + + Ok(Self { + index, + writer: Mutex::new(writer), + reader, + record_id_field, + message_field, + }) + } + + /// Upserts one record: delete-then-add on record_id. Tantivy segments + /// are immutable, so this delete-then-add is the standard idiom for + /// updates, not a workaround -- and it matters here specifically + /// because /search's offset tracking is best-effort (see + /// consumer.rs's OffsetStore), so the same record can genuinely be + /// reprocessed after an unclean shutdown. Without this, that would + /// silently duplicate documents instead of just re-writing the same + /// one. + pub async fn upsert(&self, record_id: &str, message: &str) -> Result<()> { + let writer = self.writer.lock().await; + let term = Term::from_field_text(self.record_id_field, record_id); + writer.delete_term(term); + writer + .add_document(doc!( + self.record_id_field => record_id, + self.message_field => message, + )) + .context("adding document to tantivy index")?; + Ok(()) + } + + pub async fn commit(&self) -> Result<()> { + let mut writer = self.writer.lock().await; + writer.commit().context("committing tantivy index")?; + // Explicit reload rather than relying solely on ReloadPolicy:: + // OnCommitWithDelay's background timing: callers of `commit()` + // (the periodic ticker in consumer.rs, and tests) expect a + // committed document to be immediately searchable, not visible + // after some undocumented delay. + self.reader.reload().context("reloading tantivy reader after commit")?; + Ok(()) + } + + /// Runs a Tantivy query-parser query against the `message` field, + /// returning matching record_ids, most-relevant first. Phase 1: no + /// pagination, no score exposed to the caller — just IDs for `/api` + /// to join against ClickHouse. + pub fn search(&self, query: &str, limit: usize) -> Result> { + let searcher = self.reader.searcher(); + let query_parser = QueryParser::for_index(&self.index, vec![self.message_field]); + let parsed_query = query_parser + .parse_query(query) + .context("parsing search query")?; + let top_docs = searcher + .search(&parsed_query, &TopDocs::with_limit(limit)) + .context("executing search")?; + + let mut ids = Vec::with_capacity(top_docs.len()); + for (_score, doc_address) in top_docs { + let retrieved: TantivyDocument = searcher + .doc(doc_address) + .context("retrieving matched document")?; + if let Some(value) = retrieved.get_first(self.record_id_field) { + if let Some(s) = value.as_str() { + ids.push(s.to_string()); + } + } + } + Ok(ids) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn new_test_index() -> (SearchIndex, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("creating temp dir"); + let index = SearchIndex::open_or_create(dir.path()).expect("opening tantivy index"); + (index, dir) + } + + #[tokio::test] + async fn upsert_and_search_finds_matching_message() { + let (index, _dir) = new_test_index(); + index.upsert("id-1", "hello world").await.unwrap(); + index.upsert("id-2", "goodbye moon").await.unwrap(); + index.commit().await.unwrap(); + + let results = index.search("hello", 10).unwrap(); + assert_eq!(results, vec!["id-1".to_string()]); + } + + #[tokio::test] + async fn search_before_commit_finds_nothing() { + let (index, _dir) = new_test_index(); + index.upsert("id-1", "hello world").await.unwrap(); + // no commit yet + let results = index.search("hello", 10).unwrap(); + assert!(results.is_empty(), "expected no results before commit, got {results:?}"); + } + + #[tokio::test] + async fn upsert_same_id_twice_does_not_duplicate() { + let (index, _dir) = new_test_index(); + index.upsert("id-1", "hello world").await.unwrap(); + index.commit().await.unwrap(); + index.upsert("id-1", "hello world again").await.unwrap(); + index.commit().await.unwrap(); + + let results = index.search("hello", 10).unwrap(); + assert_eq!( + results.len(), + 1, + "expected exactly one result after re-upserting the same record_id, got {results:?}" + ); + } + + #[tokio::test] + async fn search_respects_limit() { + let (index, _dir) = new_test_index(); + for i in 0..5 { + index + .upsert(&format!("id-{i}"), "shared term") + .await + .unwrap(); + } + index.commit().await.unwrap(); + + let results = index.search("shared", 2).unwrap(); + assert_eq!(results.len(), 2); + } + + #[tokio::test] + async fn search_supports_phrase_queries() { + let (index, _dir) = new_test_index(); + index.upsert("id-1", "the quick brown fox").await.unwrap(); + index.upsert("id-2", "quick and brown but not adjacent fox").await.unwrap(); + index.commit().await.unwrap(); + + let results = index.search("\"quick brown\"", 10).unwrap(); + assert_eq!(results, vec!["id-1".to_string()]); + } +} diff --git a/search/src/main.rs b/search/src/main.rs new file mode 100644 index 0000000..8a93e0c --- /dev/null +++ b/search/src/main.rs @@ -0,0 +1,67 @@ +mod config; +mod consumer; +mod grpc; +mod index; +mod offsets; + +pub mod logsv1 { + tonic::include_proto!("sentry.logs.v1"); +} +pub mod searchv1 { + tonic::include_proto!("sentry.search.v1"); +} + +use anyhow::{Context, Result}; +use config::Config; +use index::SearchIndex; +use std::sync::Arc; +use tonic::transport::Server; + +/// Matches /transport/provision-topics.sh's default +/// REDPANDA_TOPIC_PARTITIONS -- documented cross-component contract, not +/// discovered dynamically. See consumer.rs. +const DEFAULT_PARTITION_COUNT: i32 = 6; + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .init(); + + let cfg = Arc::new(Config::load().context("loading config")?); + + let index = Arc::new( + SearchIndex::open_or_create(&cfg.index_path).context("opening tantivy index")?, + ); + + let partition_count: i32 = std::env::var("REDPANDA_TOPIC_PARTITIONS") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(DEFAULT_PARTITION_COUNT); + + let consumer_cfg = Arc::clone(&cfg); + let consumer_index = Arc::clone(&index); + let consumer_handle = tokio::spawn(async move { + if let Err(e) = consumer::run(consumer_cfg, consumer_index, partition_count).await { + tracing::error!(error = %e, "redpanda consumer exited with error"); + } + }); + + let addr = cfg + .grpc_listen_addr + .parse() + .context("parsing GRPC_LISTEN_ADDR")?; + tracing::info!(addr = %cfg.grpc_listen_addr, "search gRPC server listening"); + + let search_server = grpc::SearchServer::new(Arc::clone(&index)); + Server::builder() + .add_service(searchv1::search_service_server::SearchServiceServer::new( + search_server, + )) + .serve(addr) + .await + .context("gRPC server failed")?; + + consumer_handle.abort(); + Ok(()) +} diff --git a/search/src/offsets.rs b/search/src/offsets.rs new file mode 100644 index 0000000..1e2a920 --- /dev/null +++ b/search/src/offsets.rs @@ -0,0 +1,93 @@ +use anyhow::{Context, Result}; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +/// Tracks per-partition offsets in a plain JSON file next to the Tantivy +/// index, since rskafka is a low-level client with no built-in consumer- +/// group coordination/offset-commit protocol (unlike kafka-go on the +/// ingest side) -- there's no broker-side group to commit to here, so +/// this service owns its own offset bookkeeping. +/// +/// Best-effort, not exactly-once: if the process dies between processing +/// a record and persisting its offset, that record gets reprocessed on +/// restart. This is fine because `SearchIndex::upsert` is delete-then-add +/// on `record_id` -- reprocessing the same record overwrites the same +/// document rather than duplicating it. +#[derive(Debug)] +pub struct OffsetStore { + path: PathBuf, + offsets: HashMap, +} + +impl OffsetStore { + pub async fn load(path: &Path) -> Result { + let offsets = match tokio::fs::read(path).await { + Ok(bytes) => serde_json::from_slice(&bytes) + .with_context(|| format!("parsing offsets file {}", path.display()))?, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => HashMap::new(), + Err(e) => return Err(e).with_context(|| format!("reading offsets file {}", path.display())), + }; + Ok(Self { + path: path.to_path_buf(), + offsets, + }) + } + + /// Next offset to fetch for a partition -- 0 (earliest) if never + /// recorded before. + pub fn get(&self, partition: i32) -> i64 { + self.offsets.get(&partition).copied().unwrap_or(0) + } + + pub fn set(&mut self, partition: i32, offset: i64) { + self.offsets.insert(partition, offset); + } + + pub async fn persist(&self) -> Result<()> { + if let Some(parent) = self.path.parent() { + tokio::fs::create_dir_all(parent) + .await + .with_context(|| format!("creating offsets directory {}", parent.display()))?; + } + let bytes = serde_json::to_vec_pretty(&self.offsets).context("serializing offsets")?; + tokio::fs::write(&self.path, bytes) + .await + .with_context(|| format!("writing offsets file {}", self.path.display())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn get_defaults_to_zero_for_unknown_partition() { + let dir = tempfile::tempdir().unwrap(); + let store = OffsetStore::load(&dir.path().join("offsets.json")).await.unwrap(); + assert_eq!(store.get(0), 0); + } + + #[tokio::test] + async fn missing_file_loads_as_empty_not_an_error() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("does-not-exist.json"); + let store = OffsetStore::load(&path).await; + assert!(store.is_ok(), "expected a missing offsets file to load as empty, got {store:?}"); + } + + #[tokio::test] + async fn persist_and_reload_round_trips() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("offsets.json"); + + let mut store = OffsetStore::load(&path).await.unwrap(); + store.set(0, 42); + store.set(1, 7); + store.persist().await.unwrap(); + + let reloaded = OffsetStore::load(&path).await.unwrap(); + assert_eq!(reloaded.get(0), 42); + assert_eq!(reloaded.get(1), 7); + assert_eq!(reloaded.get(2), 0, "unset partition should still default to 0"); + } +} diff --git a/storage/README.md b/storage/README.md index a05ff5b..a557166 100644 --- a/storage/README.md +++ b/storage/README.md @@ -4,7 +4,8 @@ ClickHouse schema and migration tooling for Sentry's analytical store. ## Schema -One table for Phase 0, `logs`: +One table, `logs` (Phase 0 columns plus `record_id`, added in +`migrations/0002_add_record_id.sql`): ```sql CREATE TABLE logs @@ -14,13 +15,31 @@ CREATE TABLE logs `service` String, `severity` LowCardinality(String), `message` String, - `attributes` Map(String, String) + `attributes` Map(String, String), + `record_id` UUID DEFAULT generateUUIDv4() ) ENGINE = MergeTree PARTITION BY toDate(timestamp) ORDER BY (service, timestamp) +-- plus: INDEX record_id_idx record_id TYPE bloom_filter GRANULARITY 4 ``` +**`record_id`** (Phase 1) is the stable per-record identifier Tantivy's +full-text search joins back to this table with — `/ingest`'s gRPC front +end assigns it once, server-side, before a record is produced to +Redpanda (see `/ingest/README.md` for why it has to happen exactly once, +upstream of both the ClickHouse-writer and Tantivy-indexer consumers). +Added via `ALTER TABLE ... ADD COLUMN` + `ADD INDEX` rather than changing +`ORDER BY`: `ORDER BY (service, timestamp)` is the proven time-range-scan +access pattern from Phase 0 and shouldn't be disturbed for a +fundamentally different access pattern (point lookups by ID). A +data-skipping bloom filter index on `record_id` serves the `WHERE +record_id IN (...)` lookup Tantivy-backed search results need, without +touching the primary sort order. The `DEFAULT generateUUIDv4()` is a +safety net, not the normal path — every row `/ingest` writes explicitly +supplies its own `record_id` from the proto message; the default only +matters for rows written some other way. + Notes on choices that weren't fully specified by the task description: - **`DateTime64(9, 'UTC')`** (nanosecond precision) rather than second or diff --git a/storage/migrations/0002_add_record_id.sql b/storage/migrations/0002_add_record_id.sql new file mode 100644 index 0000000..2111421 --- /dev/null +++ b/storage/migrations/0002_add_record_id.sql @@ -0,0 +1,3 @@ +ALTER TABLE logs + ADD COLUMN record_id UUID DEFAULT generateUUIDv4(), + ADD INDEX record_id_idx record_id TYPE bloom_filter GRANULARITY 4 diff --git a/web/src/lib/ResultsTable.svelte b/web/src/lib/ResultsTable.svelte new file mode 100644 index 0000000..8ea1720 --- /dev/null +++ b/web/src/lib/ResultsTable.svelte @@ -0,0 +1,59 @@ + + +{#if hasRun} +

{rows.length} row(s)

+{/if} + +{#if columns.length > 0} + + + + {#each columns as col (col)} + + {/each} + + + + {#each rows as row, i (i)} + + {#each row as cell, j (j)} + + {/each} + + {/each} + +
{col}
{formatCell(cell)}
+{/if} + + diff --git a/web/src/routes/+layout.svelte b/web/src/routes/+layout.svelte index 8098bbb..8161d42 100644 --- a/web/src/routes/+layout.svelte +++ b/web/src/routes/+layout.svelte @@ -1,5 +1,6 @@ @@ -9,4 +10,32 @@ + + {@render children()} + + diff --git a/web/src/routes/+page.svelte b/web/src/routes/+page.svelte index 3164d03..6302290 100644 --- a/web/src/routes/+page.svelte +++ b/web/src/routes/+page.svelte @@ -4,6 +4,8 @@ // This is a placeholder for the real query UI that lands once /api grows // a real SPL-like query layer in Phase 2. + import ResultsTable from '$lib/ResultsTable.svelte'; + const apiBase = import.meta.env.VITE_API_BASE_URL ?? 'http://localhost:8080'; let sql = $state('SELECT * FROM logs ORDER BY timestamp DESC LIMIT 100'); @@ -13,12 +15,6 @@ let loading = $state(false); let hasRun = $state(false); - function formatCell(value: unknown): string { - if (value === null || value === undefined) return ''; - if (typeof value === 'object') return JSON.stringify(value); - return String(value); - } - async function runQuery() { loading = true; error = ''; @@ -49,10 +45,11 @@
-

Sentry — Log Query (Phase 0)

+

Sentry — Log Query

Raw SQL only, SELECT statements against the logs table. No auth, no query - builder yet — see /api for what's actually allowed. + builder yet — see /api for what's actually allowed. Looking for free-text + search instead? See the Full-Text Search page.

@@ -66,30 +63,7 @@

Error: {error}

{/if} - {#if hasRun && !error} -

{rows.length} row(s)

- {/if} - - {#if columns.length > 0} - - - - {#each columns as col (col)} - - {/each} - - - - {#each rows as row, i (i)} - - {#each row as cell, j (j)} - - {/each} - - {/each} - -
{col}
{formatCell(cell)}
- {/if} +
diff --git a/web/src/routes/search/+page.svelte b/web/src/routes/search/+page.svelte new file mode 100644 index 0000000..03187b9 --- /dev/null +++ b/web/src/routes/search/+page.svelte @@ -0,0 +1,96 @@ + + +
+

Sentry — Full-Text Search

+

+ Free-text search over the message field, via Tantivy. Supports plain terms, + "exact phrases", and wildcard* — see + /search for the full query syntax. Looking for structured/aggregation queries + instead? See the SQL Query page. +

+ + e.key === 'Enter' && runSearch()} + /> +
+ +
+ + {#if error} +

Error: {error}

+ {/if} + + +
+ + diff --git a/web/src/routes/search/+page.ts b/web/src/routes/search/+page.ts new file mode 100644 index 0000000..8a3e476 --- /dev/null +++ b/web/src/routes/search/+page.ts @@ -0,0 +1,4 @@ +// Same reasoning as the root page's +page.ts: no load function (all data +// comes from a client-side fetch on submit), so a plain prerender is +// enough for the static adapter. +export const prerender = true;