coffey-labs/cairnobs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Phase 1: Windows log collection + full-text search

Browse Source 
Extends the agent, ingest, storage, api, and web with Windows Event
Log/ETW sourcing and Tantivy-backed free-text search, per the approved
Phase 1 plan.

- CLAUDE.md: materialized on disk (never existed as a file before) with
  a new Phase 1 "done looks like" section.
- agent: Windows Event Log (EvtSubscribe) and ETW sources, Windows
  service wrapper (install/uninstall/run-service), both feature- and
  target_os-gated so Linux builds/tests/clippy stay unaffected. Also
  fixed two pre-existing Phase 0 clippy gaps (dead-code on
  default-features-only builds, a type-inference edge case) found while
  testing every feature combination properly for the first time.
  UNVERIFIED on real Windows -- no Windows toolchain existed anywhere in
  the build environment; flagged prominently in three places.
- proto/ingest: new record_id field, assigned once server-side in
  ingest's gRPC front end so ClickHouse and Tantivy agree on the same ID
  for the same record.
- storage: record_id column + bloom filter index, verified against a
  live ClickHouse.
- search: new service, Tantivy index, rskafka consumer as an independent
  second consumer group on the same Redpanda topic ingest already reads.
- api/web: new /search endpoint and page, sharing the query page's
  result-table shape and component.
- hack/windows-fixture: sends realistic Windows-shaped data straight to
  ingest, so the pipeline's handling of it is verifiable without a
  Windows host.

Verified end-to-end on the live docker-compose stack: the same record_id
comes back from both /query and /search for the same log line, including
for windows-fixture's synthetic Windows Event Log data. Real bugs found
and fixed along the way: api/Dockerfile missing proto/ in its build
context, search's logs being completely silent (RUST_LOG gap), and
search/target/ missing from .gitignore/.dockerignore.
This commit is contained in:
jcoffey-dev
2026-08-13 11:27:35 -07:00
parent fe854b1091
commit cd8aa290ca
66 changed files with 6084 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files
proto/sentry/logs/v1/logs.pb.go
+22 -4
View File
@@ -103,8 +103,18 @@ type LogRecord struct {
// requirement in CLAUDE.md.
Message string `protobuf:"bytes,5,opt,name=message,proto3" json:"message,omitempty"`
// Structured fields extracted by the agent's parser (e.g. RFC 5424
// syslog header fields). Empty when the raw-passthrough fallback fires.
Attributes map[string]string `protobuf:"bytes,6,rep,name=attributes,proto3" json:"attributes,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
// syslog header fields), plus source-provided fields (e.g. Windows
// Event Log's winevt.event_id/winevt.provider/winevt.channel). Empty
// when the raw-passthrough fallback fires and the source added nothing.
Attributes map[string]string `protobuf:"bytes,6,rep,name=attributes,proto3" json:"attributes,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
// Stable per-record identifier, used to join Tantivy full-text search
// hits back to their ClickHouse row (Phase 1). Always empty as sent by
// the agent — ingest's PushBatch handler assigns this server-side,
// once, before producing to Redpanda, since both the ClickHouse-writer
// consumer and the Tantivy-indexer consumer read the same Redpanda
// messages and need to agree on the same ID for the same record. See
// /ingest/README.md.
RecordId string `protobuf:"bytes,7,opt,name=record_id,json=recordId,proto3" json:"record_id,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -181,6 +191,13 @@ func (x *LogRecord) GetAttributes() map[string]string {
return nil
}
func (x *LogRecord) GetRecordId() string {
if x != nil {
return x.RecordId
}
return ""
}
type PushBatchRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
// Agent-assigned identifier for dedup/idempotency on retry. Ingest may
@@ -287,7 +304,7 @@ var File_sentry_logs_v1_logs_proto protoreflect.FileDescriptor
const file_sentry_logs_v1_logs_proto_rawDesc = "" +
"\n" +
"\x19sentry/logs/v1/logs.proto\x12\x0esentry.logs.v1\"\xc3\x02\n" +
"\x19sentry/logs/v1/logs.proto\x12\x0esentry.logs.v1\"\xe0\x02\n" +
"\tLogRecord\x12.\n" +
"\x13timestamp_unix_nano\x18\x01 \x01(\x03R\x11timestampUnixNano\x12\x12\n" +
"\x04host\x18\x02 \x01(\tR\x04host\x12\x18\n" +
@@ -296,7 +313,8 @@ const file_sentry_logs_v1_logs_proto_rawDesc = "" +
"\amessage\x18\x05 \x01(\tR\amessage\x12I\n" +
"\n" +
"attributes\x18\x06 \x03(\v2).sentry.logs.v1.LogRecord.AttributesEntryR\n" +
"attributes\x1a=\n" +
"attributes\x12\x1b\n" +
"\trecord_id\x18\a \x01(\tR\brecordId\x1a=\n" +
"\x0fAttributesEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"b\n" +
proto/sentry/logs/v1/logs.proto
+12 -1
View File
@@ -47,8 +47,19 @@ message LogRecord {
string message = 5;
// Structured fields extracted by the agent's parser (e.g. RFC 5424
// syslog header fields). Empty when the raw-passthrough fallback fires.
// syslog header fields), plus source-provided fields (e.g. Windows
// Event Log's winevt.event_id/winevt.provider/winevt.channel). Empty
// when the raw-passthrough fallback fires and the source added nothing.
map<string, string> attributes = 6;
// Stable per-record identifier, used to join Tantivy full-text search
// hits back to their ClickHouse row (Phase 1). Always empty as sent by
// the agent — ingest's PushBatch handler assigns this server-side,
// once, before producing to Redpanda, since both the ClickHouse-writer
// consumer and the Tantivy-indexer consumer read the same Redpanda
// messages and need to agree on the same ID for the same record. See
// /ingest/README.md.
string record_id = 7;
}
message PushBatchRequest {
proto/sentry/search/v1/search.pb.go
+192
View File
@@ -0,0 +1,192 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.36.12
// protoc v7.35.1
// source: sentry/search/v1/search.proto
package searchv1
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
sync "sync"
unsafe "unsafe"
)
const (
// Verify that this generated code is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
// Verify that runtime/protoimpl is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
type SearchRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
// Free-text query passed to Tantivy's query parser as-is. Supports
// phrase queries ("exact phrase") and wildcards (foo*) per Tantivy's
// own query syntax — see /search/README.md for exactly what that does
// and doesn't support in Phase 1.
Query string `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"`
// Max results to return. 0 (unset) uses the service's own default.
Limit uint32 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *SearchRequest) Reset() {
*x = SearchRequest{}
mi := &file_sentry_search_v1_search_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *SearchRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*SearchRequest) ProtoMessage() {}
func (x *SearchRequest) ProtoReflect() protoreflect.Message {
mi := &file_sentry_search_v1_search_proto_msgTypes[0]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use SearchRequest.ProtoReflect.Descriptor instead.
func (*SearchRequest) Descriptor() ([]byte, []int) {
return file_sentry_search_v1_search_proto_rawDescGZIP(), []int{0}
}
func (x *SearchRequest) GetQuery() string {
if x != nil {
return x.Query
}
return ""
}
func (x *SearchRequest) GetLimit() uint32 {
if x != nil {
return x.Limit
}
return 0
}
type SearchResponse struct {
state protoimpl.MessageState `protogen:"open.v1"`
// record_ids of matching logs, most-relevant first. Callers join these
// back against ClickHouse's `logs.record_id` column to get full rows —
// this service only ever returns IDs, never row data, so it stays a
// pure text index rather than a second copy of the row.
RecordIds []string `protobuf:"bytes,1,rep,name=record_ids,json=recordIds,proto3" json:"record_ids,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *SearchResponse) Reset() {
*x = SearchResponse{}
mi := &file_sentry_search_v1_search_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *SearchResponse) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*SearchResponse) ProtoMessage() {}
func (x *SearchResponse) ProtoReflect() protoreflect.Message {
mi := &file_sentry_search_v1_search_proto_msgTypes[1]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use SearchResponse.ProtoReflect.Descriptor instead.
func (*SearchResponse) Descriptor() ([]byte, []int) {
return file_sentry_search_v1_search_proto_rawDescGZIP(), []int{1}
}
func (x *SearchResponse) GetRecordIds() []string {
if x != nil {
return x.RecordIds
}
return nil
}
var File_sentry_search_v1_search_proto protoreflect.FileDescriptor
const file_sentry_search_v1_search_proto_rawDesc = "" +
"\n" +
"\x1dsentry/search/v1/search.proto\x12\x10sentry.search.v1\";\n" +
"\rSearchRequest\x12\x14\n" +
"\x05query\x18\x01 \x01(\tR\x05query\x12\x14\n" +
"\x05limit\x18\x02 \x01(\rR\x05limit\"/\n" +
"\x0eSearchResponse\x12\x1d\n" +
"\n" +
"record_ids\x18\x01 \x03(\tR\trecordIds2\\\n" +
"\rSearchService\x12K\n" +
"\x06Search\x12\x1f.sentry.search.v1.SearchRequest\x1a .sentry.search.v1.SearchResponseB:Z8github.com/sentry/sentry/proto/sentry/search/v1;searchv1b\x06proto3"
var (
file_sentry_search_v1_search_proto_rawDescOnce sync.Once
file_sentry_search_v1_search_proto_rawDescData []byte
)
func file_sentry_search_v1_search_proto_rawDescGZIP() []byte {
file_sentry_search_v1_search_proto_rawDescOnce.Do(func() {
file_sentry_search_v1_search_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_sentry_search_v1_search_proto_rawDesc), len(file_sentry_search_v1_search_proto_rawDesc)))
})
return file_sentry_search_v1_search_proto_rawDescData
}
var file_sentry_search_v1_search_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
var file_sentry_search_v1_search_proto_goTypes = []any{
(*SearchRequest)(nil), // 0: sentry.search.v1.SearchRequest
(*SearchResponse)(nil), // 1: sentry.search.v1.SearchResponse
}
var file_sentry_search_v1_search_proto_depIdxs = []int32{
0, // 0: sentry.search.v1.SearchService.Search:input_type -> sentry.search.v1.SearchRequest
1, // 1: sentry.search.v1.SearchService.Search:output_type -> sentry.search.v1.SearchResponse
1, // [1:2] is the sub-list for method output_type
0, // [0:1] is the sub-list for method input_type
0, // [0:0] is the sub-list for extension type_name
0, // [0:0] is the sub-list for extension extendee
0, // [0:0] is the sub-list for field type_name
}
func init() { file_sentry_search_v1_search_proto_init() }
func file_sentry_search_v1_search_proto_init() {
if File_sentry_search_v1_search_proto != nil {
return
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_sentry_search_v1_search_proto_rawDesc), len(file_sentry_search_v1_search_proto_rawDesc)),
NumEnums: 0,
NumMessages: 2,
NumExtensions: 0,
NumServices: 1,
},
GoTypes: file_sentry_search_v1_search_proto_goTypes,
DependencyIndexes: file_sentry_search_v1_search_proto_depIdxs,
MessageInfos: file_sentry_search_v1_search_proto_msgTypes,
}.Build()
File_sentry_search_v1_search_proto = out.File
file_sentry_search_v1_search_proto_goTypes = nil
file_sentry_search_v1_search_proto_depIdxs = nil
}
proto/sentry/search/v1/search.proto
+33
View File
@@ -0,0 +1,33 @@
syntax = "proto3";
package sentry.search.v1;
option go_package = "github.com/sentry/sentry/proto/sentry/search/v1;searchv1";
// SearchService is the full-text search index (Tantivy-backed) that
// `api` calls to resolve a free-text query into matching record_ids,
// which `api` then joins back against ClickHouse. Internal service-to-
// service call, same gRPC-first convention as agent<->ingest — see
// /docs/architecture.md and /search/README.md.
service SearchService {
rpc Search(SearchRequest) returns (SearchResponse);
}
message SearchRequest {
// Free-text query passed to Tantivy's query parser as-is. Supports
// phrase queries ("exact phrase") and wildcards (foo*) per Tantivy's
// own query syntax — see /search/README.md for exactly what that does
// and doesn't support in Phase 1.
string query = 1;
// Max results to return. 0 (unset) uses the service's own default.
uint32 limit = 2;
}
message SearchResponse {
// record_ids of matching logs, most-relevant first. Callers join these
// back against ClickHouse's `logs.record_id` column to get full rows —
// this service only ever returns IDs, never row data, so it stays a
// pure text index rather than a second copy of the row.
repeated string record_ids = 1;
}
proto/sentry/search/v1/search_grpc.pb.go
+133
View File
@@ -0,0 +1,133 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.6.2
// - protoc v7.35.1
// source: sentry/search/v1/search.proto
package searchv1
import (
context "context"
grpc "google.golang.org/grpc"
codes "google.golang.org/grpc/codes"
status "google.golang.org/grpc/status"
)
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.64.0 or later.
const _ = grpc.SupportPackageIsVersion9
const (
SearchService_Search_FullMethodName = "/sentry.search.v1.SearchService/Search"
)
// SearchServiceClient is the client API for SearchService service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
//
// SearchService is the full-text search index (Tantivy-backed) that
// `api` calls to resolve a free-text query into matching record_ids,
// which `api` then joins back against ClickHouse. Internal service-to-
// service call, same gRPC-first convention as agent<->ingest — see
// /docs/architecture.md and /search/README.md.
type SearchServiceClient interface {
Search(ctx context.Context, in *SearchRequest, opts ...grpc.CallOption) (*SearchResponse, error)
}
type searchServiceClient struct {
cc grpc.ClientConnInterface
}
func NewSearchServiceClient(cc grpc.ClientConnInterface) SearchServiceClient {
return &searchServiceClient{cc}
}
func (c *searchServiceClient) Search(ctx context.Context, in *SearchRequest, opts ...grpc.CallOption) (*SearchResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(SearchResponse)
err := c.cc.Invoke(ctx, SearchService_Search_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
// SearchServiceServer is the server API for SearchService service.
// All implementations must embed UnimplementedSearchServiceServer
// for forward compatibility.
//
// SearchService is the full-text search index (Tantivy-backed) that
// `api` calls to resolve a free-text query into matching record_ids,
// which `api` then joins back against ClickHouse. Internal service-to-
// service call, same gRPC-first convention as agent<->ingest — see
// /docs/architecture.md and /search/README.md.
type SearchServiceServer interface {
Search(context.Context, *SearchRequest) (*SearchResponse, error)
mustEmbedUnimplementedSearchServiceServer()
}
// UnimplementedSearchServiceServer must be embedded to have
// forward compatible implementations.
//
// NOTE: this should be embedded by value instead of pointer to avoid a nil
// pointer dereference when methods are called.
type UnimplementedSearchServiceServer struct{}
func (UnimplementedSearchServiceServer) Search(context.Context, *SearchRequest) (*SearchResponse, error) {
return nil, status.Error(codes.Unimplemented, "method Search not implemented")
}
func (UnimplementedSearchServiceServer) mustEmbedUnimplementedSearchServiceServer() {}
func (UnimplementedSearchServiceServer) testEmbeddedByValue() {}
// UnsafeSearchServiceServer may be embedded to opt out of forward compatibility for this service.
// Use of this interface is not recommended, as added methods to SearchServiceServer will
// result in compilation errors.
type UnsafeSearchServiceServer interface {
mustEmbedUnimplementedSearchServiceServer()
}
func RegisterSearchServiceServer(s grpc.ServiceRegistrar, srv SearchServiceServer) {
// If the following call panics, it indicates UnimplementedSearchServiceServer was
// embedded by pointer and is nil. This will cause panics if an
// unimplemented method is ever invoked, so we test this at initialization
// time to prevent it from happening at runtime later due to I/O.
if t, ok := srv.(interface{ testEmbeddedByValue() }); ok {
t.testEmbeddedByValue()
}
s.RegisterService(&SearchService_ServiceDesc, srv)
}
func _SearchService_Search_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(SearchRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(SearchServiceServer).Search(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: SearchService_Search_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(SearchServiceServer).Search(ctx, req.(*SearchRequest))
}
return interceptor(ctx, in, info, handler)
}
// SearchService_ServiceDesc is the grpc.ServiceDesc for SearchService service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
var SearchService_ServiceDesc = grpc.ServiceDesc{
ServiceName: "sentry.search.v1.SearchService",
HandlerType: (*SearchServiceServer)(nil),
Methods: []grpc.MethodDesc{
{
MethodName: "Search",
Handler: _SearchService_Search_Handler,
},
},
Streams: []grpc.StreamDesc{},
Metadata: "sentry/search/v1/search.proto",
}