Scaffold Phase 0: agent -> Redpanda -> ingest -> ClickHouse -> api -> web

End-to-end log pipeline for Linux hosts, per /docs/architecture.md: - proto: shared gRPC contract (agent <-> ingest), Go bindings checked in - agent: Rust, musl-targeted, journald/file sourcing, RFC5424 parser, mTLS gRPC client, no required config for the common case - ingest: Go, single binary with --mode server|consumer|all; gRPC front end forwards to Redpanda unchanged, consumer normalizes and batch-writes to ClickHouse with at-least-once delivery - storage: ClickHouse schema + a plain SQL-file migration runner - api: minimal SELECT-only query endpoint, plain REST (not gRPC+gateway yet -- see api/README.md) - web: SvelteKit static SPA, one query page - transport: Redpanda compose + topic provisioning - cli: sentryctl ping stub - hack/dev-certs: throwaway CA + cert generation for local mTLS - root docker-compose.yml + docs/phase-0-runbook.md tie it together Not yet run end-to-end against real Docker/ClickHouse/Redpanda -- see the runbook's caveats section before relying on this working as-is.
2026-08-13 08:25:19 -07:00
commit b6b092c912
92 changed files with 7796 additions and 0 deletions
@@ -0,0 +1,56 @@
+// Package config loads api's configuration from environment variables,
+// same convention as /ingest: no config file format for Phase 0.
+package config
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"time"
+)
+
+type Config struct {
+	HTTPListenAddr    string
+	ClickHouse        ClickHouseConfig
+	QueryTimeout      time.Duration
+	CORSAllowedOrigin string
+}
+
+type ClickHouseConfig struct {
+	Addr     string
+	Database string
+	Username string
+	Password string
+}
+
+func Load() (Config, error) {
+	cfg := Config{
+		HTTPListenAddr: getenv("HTTP_LISTEN_ADDR", ":8080"),
+		ClickHouse: ClickHouseConfig{
+			Addr:     getenv("CLICKHOUSE_ADDR", "localhost:9000"),
+			Database: getenv("CLICKHOUSE_DATABASE", "sentry"),
+			Username: getenv("CLICKHOUSE_USERNAME", "default"),
+			Password: getenv("CLICKHOUSE_PASSWORD", ""),
+		},
+		// Phase 0 has no auth, so this is wide open by default to keep
+		// the local SvelteKit dev server (a different origin/port)
+		// working out of the box. Tighten before this is ever reachable
+		// from outside a trusted dev/homelab network.
+		CORSAllowedOrigin: getenv("CORS_ALLOWED_ORIGIN", "*"),
+	}
+
+	timeoutSec, err := strconv.Atoi(getenv("QUERY_TIMEOUT_SECONDS", "30"))
+	if err != nil {
+		return Config{}, fmt.Errorf("QUERY_TIMEOUT_SECONDS: %w", err)
+	}
+	cfg.QueryTimeout = time.Duration(timeoutSec) * time.Second
+
+	return cfg, nil
+}
+
+func getenv(key, fallback string) string {
+	if v := os.Getenv(key); v != "" {
+		return v
+	}
+	return fallback
+}
@@ -0,0 +1,29 @@
+package config
+
+import (
+	"testing"
+	"time"
+)
+
+func TestLoadDefaults(t *testing.T) {
+	cfg, err := Load()
+	if err != nil {
+		t.Fatalf("Load() error = %v", err)
+	}
+	if cfg.HTTPListenAddr != ":8080" {
+		t.Errorf("HTTPListenAddr = %q, want :8080", cfg.HTTPListenAddr)
+	}
+	if cfg.QueryTimeout != 30*time.Second {
+		t.Errorf("QueryTimeout = %v, want 30s", cfg.QueryTimeout)
+	}
+	if cfg.CORSAllowedOrigin != "*" {
+		t.Errorf("CORSAllowedOrigin = %q, want *", cfg.CORSAllowedOrigin)
+	}
+}
+
+func TestLoadInvalidTimeoutErrors(t *testing.T) {
+	t.Setenv("QUERY_TIMEOUT_SECONDS", "not-a-number")
+	if _, err := Load(); err == nil {
+		t.Fatal("expected error for non-numeric QUERY_TIMEOUT_SECONDS, got nil")
+	}
+}
@@ -0,0 +1,60 @@
+package queryapi
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+
+	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+)
+
+type QueryResult struct {
+	Columns []string `json:"columns"`
+	Rows    [][]any  `json:"rows"`
+}
+
+// Executor runs arbitrary (pre-validated) SELECT statements against
+// ClickHouse and shapes the result into JSON-friendly columns/rows,
+// discovering the result's column set at query time via reflection since
+// the query itself is arbitrary.
+type Executor struct {
+	conn driver.Conn
+}
+
+func NewExecutor(conn driver.Conn) *Executor {
+	return &Executor{conn: conn}
+}
+
+func (e *Executor) Execute(ctx context.Context, sql string) (*QueryResult, error) {
+	rows, err := e.conn.Query(ctx, sql)
+	if err != nil {
+		return nil, fmt.Errorf("executing query: %w", err)
+	}
+	defer rows.Close()
+
+	columnTypes := rows.ColumnTypes()
+	result := &QueryResult{
+		Columns: rows.Columns(),
+		Rows:    [][]any{},
+	}
+
+	for rows.Next() {
+		dest := make([]any, len(columnTypes))
+		for i, ct := range columnTypes {
+			dest[i] = reflect.New(ct.ScanType()).Interface()
+		}
+		if err := rows.Scan(dest...); err != nil {
+			return nil, fmt.Errorf("scanning row: %w", err)
+		}
+		row := make([]any, len(dest))
+		for i, d := range dest {
+			row[i] = reflect.ValueOf(d).Elem().Interface()
+		}
+		result.Rows = append(result.Rows, row)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("iterating rows: %w", err)
+	}
+
+	return result, nil
+}
@@ -0,0 +1,112 @@
+// Package queryapi is the Phase 0 query API: a single crude POST /query
+// endpoint that takes a raw SQL string, allowlists it to a single SELECT
+// statement, and proxies it to ClickHouse. This is a deliberate
+// simplification of the pinned "gRPC + REST gateway" control-plane
+// pattern (see CLAUDE.md's tech stack table): a plain net/http REST
+// handler, not a gRPC service transcoded through grpc-gateway. That
+// machinery (proto definitions, googleapis annotations, gateway codegen)
+// buys nothing for one crude placeholder endpoint that Phase 2 replaces
+// outright with the real SPL-like query layer. Revisit gRPC+gateway when
+// /api grows a second real endpoint.
+package queryapi
+
+import (
+	"context"
+	"encoding/json"
+	"log/slog"
+	"net/http"
+	"time"
+)
+
+// queryExecutor is the narrow interface handleQuery depends on, so tests
+// can substitute a fake without a real ClickHouse connection. *Executor
+// satisfies it.
+type queryExecutor interface {
+	Execute(ctx context.Context, sql string) (*QueryResult, error)
+}
+
+type Handler struct {
+	logger        *slog.Logger
+	exec          queryExecutor
+	queryTimeout  time.Duration
+	allowedOrigin string
+}
+
+func NewHandler(logger *slog.Logger, exec queryExecutor, queryTimeout time.Duration, allowedOrigin string) *Handler {
+	return &Handler{logger: logger, exec: exec, queryTimeout: queryTimeout, allowedOrigin: allowedOrigin}
+}
+
+func (h *Handler) Routes() http.Handler {
+	mux := http.NewServeMux()
+	mux.HandleFunc("POST /query", h.handleQuery)
+	mux.HandleFunc("GET /healthz", h.handleHealthz)
+	return h.withCORS(mux)
+}
+
+// withCORS is deliberately permissive by default (see CORSAllowedOrigin in
+// internal/config) since Phase 0 has no auth and the SvelteKit dev server
+// runs on a different origin. Tighten alongside adding real auth.
+func (h *Handler) withCORS(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Access-Control-Allow-Origin", h.allowedOrigin)
+		w.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS")
+		w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
+		if r.Method == http.MethodOptions {
+			w.WriteHeader(http.StatusNoContent)
+			return
+		}
+		next.ServeHTTP(w, r)
+	})
+}
+
+func (h *Handler) handleHealthz(w http.ResponseWriter, _ *http.Request) {
+	w.WriteHeader(http.StatusOK)
+}
+
+type queryRequest struct {
+	SQL string `json:"sql"`
+}
+
+type errorResponse struct {
+	Error string `json:"error"`
+}
+
+// maxBodyBytes caps the request body: a raw SQL string has no legitimate
+// reason to be larger than this.
+const maxBodyBytes = 1 << 20 // 1 MiB
+
+func (h *Handler) handleQuery(w http.ResponseWriter, r *http.Request) {
+	r.Body = http.MaxBytesReader(w, r.Body, maxBodyBytes)
+
+	var req queryRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid JSON body: "+err.Error())
+		return
+	}
+
+	if err := validateSelectOnly(req.SQL); err != nil {
+		writeError(w, http.StatusBadRequest, err.Error())
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(r.Context(), h.queryTimeout)
+	defer cancel()
+
+	result, err := h.exec.Execute(ctx, req.SQL)
+	if err != nil {
+		h.logger.Error("query execution failed", "error", err)
+		writeError(w, http.StatusBadGateway, "query failed: "+err.Error())
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(w).Encode(result); err != nil {
+		h.logger.Error("encoding response", "error", err)
+	}
+}
+
+func writeError(w http.ResponseWriter, status int, msg string) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	_ = json.NewEncoder(w).Encode(errorResponse{Error: msg})
+}
@@ -0,0 +1,134 @@
+package queryapi
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+)
+
+type fakeExecutor struct {
+	result *QueryResult
+	err    error
+	gotSQL string
+}
+
+func (f *fakeExecutor) Execute(_ context.Context, sql string) (*QueryResult, error) {
+	f.gotSQL = sql
+	if f.err != nil {
+		return nil, f.err
+	}
+	return f.result, nil
+}
+
+func newTestHandler(exec queryExecutor) *Handler {
+	return NewHandler(slog.New(slog.NewTextHandler(io.Discard, nil)), exec, time.Second, "*")
+}
+
+func TestHandleQuerySuccess(t *testing.T) {
+	fe := &fakeExecutor{result: &QueryResult{
+		Columns: []string{"host", "count"},
+		Rows:    [][]any{{"h1", 3}},
+	}}
+	h := newTestHandler(fe)
+
+	body := strings.NewReader(`{"sql": "SELECT host, count(*) FROM logs GROUP BY host"}`)
+	req := httptest.NewRequest(http.MethodPost, "/query", body)
+	rec := httptest.NewRecorder()
+
+	h.Routes().ServeHTTP(rec, req)
+
+	if rec.Code != http.StatusOK {
+		t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
+	}
+	var got QueryResult
+	if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
+		t.Fatalf("decoding response: %v", err)
+	}
+	if len(got.Columns) != 2 || len(got.Rows) != 1 {
+		t.Fatalf("unexpected result: %+v", got)
+	}
+	if fe.gotSQL != "SELECT host, count(*) FROM logs GROUP BY host" {
+		t.Fatalf("executor received unexpected SQL: %q", fe.gotSQL)
+	}
+}
+
+func TestHandleQueryRejectsNonSelect(t *testing.T) {
+	fe := &fakeExecutor{}
+	h := newTestHandler(fe)
+
+	body := strings.NewReader(`{"sql": "DELETE FROM logs"}`)
+	req := httptest.NewRequest(http.MethodPost, "/query", body)
+	rec := httptest.NewRecorder()
+
+	h.Routes().ServeHTTP(rec, req)
+
+	if rec.Code != http.StatusBadRequest {
+		t.Fatalf("status = %d, want 400", rec.Code)
+	}
+	if fe.gotSQL != "" {
+		t.Fatal("executor should not have been called for a rejected query")
+	}
+}
+
+func TestHandleQueryRejectsInvalidJSON(t *testing.T) {
+	h := newTestHandler(&fakeExecutor{})
+
+	body := strings.NewReader(`not json`)
+	req := httptest.NewRequest(http.MethodPost, "/query", body)
+	rec := httptest.NewRecorder()
+
+	h.Routes().ServeHTTP(rec, req)
+
+	if rec.Code != http.StatusBadRequest {
+		t.Fatalf("status = %d, want 400", rec.Code)
+	}
+}
+
+func TestHandleQueryExecutorErrorReturnsBadGateway(t *testing.T) {
+	fe := &fakeExecutor{err: errors.New("boom")}
+	h := newTestHandler(fe)
+
+	body := strings.NewReader(`{"sql": "SELECT 1"}`)
+	req := httptest.NewRequest(http.MethodPost, "/query", body)
+	rec := httptest.NewRecorder()
+
+	h.Routes().ServeHTTP(rec, req)
+
+	if rec.Code != http.StatusBadGateway {
+		t.Fatalf("status = %d, want 502", rec.Code)
+	}
+}
+
+func TestHandleHealthz(t *testing.T) {
+	h := newTestHandler(&fakeExecutor{})
+	req := httptest.NewRequest(http.MethodGet, "/healthz", nil)
+	rec := httptest.NewRecorder()
+
+	h.Routes().ServeHTTP(rec, req)
+
+	if rec.Code != http.StatusOK {
+		t.Fatalf("status = %d, want 200", rec.Code)
+	}
+}
+
+func TestCORSPreflight(t *testing.T) {
+	h := newTestHandler(&fakeExecutor{})
+	req := httptest.NewRequest(http.MethodOptions, "/query", nil)
+	rec := httptest.NewRecorder()
+
+	h.Routes().ServeHTTP(rec, req)
+
+	if rec.Code != http.StatusNoContent {
+		t.Fatalf("status = %d, want 204", rec.Code)
+	}
+	if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "*" {
+		t.Fatalf("Access-Control-Allow-Origin = %q, want *", got)
+	}
+}
@@ -0,0 +1,47 @@
+package queryapi
+
+import (
+	"errors"
+	"regexp"
+	"strings"
+)
+
+// disallowedKeyword is defense-in-depth on top of the SELECT-only gate: it
+// catches mutating/administrative statements appearing anywhere in the
+// query (e.g. smuggled into a subquery), not just at the start. This is
+// word-boundary matching, not a real SQL parser.
+var disallowedKeyword = regexp.MustCompile(`(?i)\b(insert|update|delete|alter|drop|truncate|create|grant|revoke|attach|detach|rename|kill|optimize|system|set|exchange|watch)\b`)
+
+// validateSelectOnly enforces the Phase 0 query API contract: exactly one
+// SELECT statement and nothing else. This is "basic injection guarding" as
+// specced, not a SQL parser: it will reject some unusual-but-valid SELECTs
+// (e.g. one that references a column literally named "delete") and will
+// not catch every possible abuse (e.g. a syntactically pure SELECT that's
+// simply expensive to run). Both are acceptable for a Phase 0 placeholder
+// that's explicitly superseded by a real query layer in Phase 2 — see
+// /docs/architecture.md.
+func validateSelectOnly(sql string) error {
+	trimmed := strings.TrimSpace(sql)
+	if trimmed == "" {
+		return errors.New("query must not be empty")
+	}
+
+	trimmed = strings.TrimSpace(strings.TrimSuffix(trimmed, ";"))
+	if trimmed == "" {
+		return errors.New("query must not be empty")
+	}
+	if strings.Contains(trimmed, ";") {
+		return errors.New("only a single statement is allowed")
+	}
+
+	firstWord := strings.ToUpper(strings.Fields(trimmed)[0])
+	if firstWord != "SELECT" {
+		return errors.New("only SELECT queries are allowed")
+	}
+
+	if disallowedKeyword.MatchString(trimmed) {
+		return errors.New("query contains a disallowed keyword")
+	}
+
+	return nil
+}
@@ -0,0 +1,38 @@
+package queryapi
+
+import "testing"
+
+func TestValidateSelectOnly(t *testing.T) {
+	cases := []struct {
+		name    string
+		sql     string
+		wantErr bool
+	}{
+		{"plain select", "SELECT * FROM logs LIMIT 10", false},
+		{"lowercase select", "select service, count(*) from logs group by service", false},
+		{"trailing semicolon allowed", "SELECT 1;", false},
+		{"trailing semicolon and whitespace allowed", "SELECT 1;  ", false},
+		{"empty", "", true},
+		{"whitespace only", "   ", true},
+		{"only a semicolon", ";", true},
+		{"multiple statements", "SELECT 1; SELECT 2", true},
+		{"insert", "INSERT INTO logs VALUES (1)", true},
+		{"delete", "DELETE FROM logs", true},
+		{"drop", "DROP TABLE logs", true},
+		{"select with drop keyword smuggled in", "SELECT * FROM logs WHERE message = 'DROP TABLE logs'", true},
+		{"non-select start", "WITH x AS (SELECT 1) SELECT * FROM x", true},
+		{"trailing garbage after semicolon", "SELECT 1; DROP TABLE logs", true},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := validateSelectOnly(tc.sql)
+			if tc.wantErr && err == nil {
+				t.Errorf("validateSelectOnly(%q) = nil, want error", tc.sql)
+			}
+			if !tc.wantErr && err != nil {
+				t.Errorf("validateSelectOnly(%q) = %v, want nil", tc.sql, err)
+			}
+		})
+	}
+}