Phase 3: dashboards and alerting

Saved, shareable multi-panel dashboards (table/line/bar/single-stat panels via gridstack + uPlot, global + per-panel time range, JSON export/import) and threshold/absence alert rules with an ok/pending/firing evaluator and webhook/Slack/PagerDuty delivery. - New /metadata component: Postgres control-plane store for dashboards, panels, notification targets, alert rules/state, and delivery log -- see docs/phase-3-dashboard-design.md for why ClickHouse's MergeTree family isn't a fit for this access pattern (needs real row-level locking and read-your-writes consistency). - api/internal/dashboards: dashboard/panel CRUD, pure -- panel query execution stays client-side, reusing the existing /query endpoint. - New /alerting service: rule/target CRUD, a ticker-driven evaluator (claim-then-evaluate concurrency control, transactional-outbox delivery, query errors and threshold zero-rows never coerced into a false transition) and webhook/Slack/PagerDuty delivery with retry/backoff. See docs/phase-3-alerting-design.md for the full state-machine design and the four correctness properties it implements. - web: /dashboards and /alerts UIs; cli: sentryctl dashboards/alerts list/get/apply, seeding a future Terraform provider's JSON contract. - hack/alert-load-test: 500 rules against real ClickHouse data, real measured results in docs/phase-3-runbook.md. Five real bugs found by actually running this against a live stack (documented in the runbook, not just fixed silently): a latent Phase 2 bug where ClickHouse rejected the timestamp format used for earliest=/latest= queries; a "now" literal token injected into query text; a GridStack/uPlot layout-timing race; JS's Date.parse being too lenient to use as a timestamp-detection heuristic; a rule's "enabled" field silently defaulting to false when omitted; and the evaluator's claim-batch-size and worker-pool-concurrency defaulting to the same value, causing 500 concurrently-due rules to take 125s to cycle through instead of the configured 60s.
2026-08-13 17:29:38 -07:00
parent fb5049a747
commit 9435115ab7
88 changed files with 7463 additions and 298 deletions
@@ -0,0 +1,261 @@
+// Package httpapi is alerting's REST surface: rule CRUD, notification
+// target CRUD, and a read-only delivery log -- mirrors
+// api/internal/dashboards' Handler/Store shape (narrow interfaces,
+// pgx-backed production implementation, fakes in tests).
+package httpapi
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"log/slog"
+	"net/http"
+
+	"github.com/sentry/sentry/alerting/internal/notifystore"
+	"github.com/sentry/sentry/alerting/internal/rulestore"
+)
+
+type ruleStore interface {
+	Create(ctx context.Context, r *rulestore.Rule) error
+	List(ctx context.Context) ([]rulestore.RuleWithState, error)
+	Get(ctx context.Context, id string) (*rulestore.RuleWithState, error)
+	Delete(ctx context.Context, id string) error
+}
+
+type targetStore interface {
+	Create(ctx context.Context, t *notifystore.Target) error
+	List(ctx context.Context) ([]notifystore.Target, error)
+	Get(ctx context.Context, id string) (*notifystore.Target, error)
+	Delete(ctx context.Context, id string) error
+}
+
+type deliveryReader interface {
+	ListForRule(ctx context.Context, ruleID string, limit int) ([]rulestore.DeliveryLogEntry, error)
+}
+
+type Handler struct {
+	logger     *slog.Logger
+	rules      ruleStore
+	targets    targetStore
+	deliveries deliveryReader
+}
+
+func NewHandler(logger *slog.Logger, rules ruleStore, targets targetStore, deliveries deliveryReader) *Handler {
+	return &Handler{logger: logger, rules: rules, targets: targets, deliveries: deliveries}
+}
+
+func (h *Handler) RegisterRoutes(mux *http.ServeMux) {
+	mux.HandleFunc("GET /healthz", h.handleHealthz)
+
+	mux.HandleFunc("POST /rules", h.handleCreateRule)
+	mux.HandleFunc("GET /rules", h.handleListRules)
+	mux.HandleFunc("GET /rules/{id}", h.handleGetRule)
+	mux.HandleFunc("DELETE /rules/{id}", h.handleDeleteRule)
+	mux.HandleFunc("GET /rules/{id}/deliveries", h.handleListDeliveries)
+
+	mux.HandleFunc("POST /targets", h.handleCreateTarget)
+	mux.HandleFunc("GET /targets", h.handleListTargets)
+	mux.HandleFunc("GET /targets/{id}", h.handleGetTarget)
+	mux.HandleFunc("DELETE /targets/{id}", h.handleDeleteTarget)
+}
+
+const maxBodyBytes = 1 << 20 // 1 MiB, same cap as api/internal/queryapi and dashboards
+
+func (h *Handler) handleHealthz(w http.ResponseWriter, _ *http.Request) {
+	w.WriteHeader(http.StatusOK)
+}
+
+// createRuleRequest mirrors rulestore.Rule except Enabled is a pointer:
+// a plain bool can't distinguish "omitted" from "explicitly false" on
+// decode, and Go's zero value for bool is false -- without this, a
+// create request that simply doesn't mention "enabled" would silently
+// create a rule the evaluator's claim query never picks up (found by
+// actually creating a rule through this endpoint and checking the
+// response). Omitted means enabled; only an explicit "enabled": false
+// creates a disabled rule.
+type createRuleRequest struct {
+	rulestore.Rule
+	Enabled *bool `json:"enabled"`
+}
+
+func (h *Handler) handleCreateRule(w http.ResponseWriter, r *http.Request) {
+	var req createRuleRequest
+	if !decodeJSON(w, r, &req) {
+		return
+	}
+	rule := req.Rule
+	rule.Enabled = req.Enabled == nil || *req.Enabled
+	if err := validateRule(&rule); err != nil {
+		writeError(w, http.StatusBadRequest, err.Error())
+		return
+	}
+	if err := h.rules.Create(r.Context(), &rule); err != nil {
+		h.logger.Error("creating rule", "error", err)
+		writeError(w, http.StatusInternalServerError, "creating rule failed")
+		return
+	}
+	writeJSON(w, http.StatusCreated, rule)
+}
+
+func (h *Handler) handleListRules(w http.ResponseWriter, r *http.Request) {
+	rules, err := h.rules.List(r.Context())
+	if err != nil {
+		h.logger.Error("listing rules", "error", err)
+		writeError(w, http.StatusInternalServerError, "listing rules failed")
+		return
+	}
+	writeJSON(w, http.StatusOK, rules)
+}
+
+func (h *Handler) handleGetRule(w http.ResponseWriter, r *http.Request) {
+	rule, err := h.rules.Get(r.Context(), r.PathValue("id"))
+	if err != nil {
+		h.writeStoreErr(w, err, "fetching rule")
+		return
+	}
+	writeJSON(w, http.StatusOK, rule)
+}
+
+func (h *Handler) handleDeleteRule(w http.ResponseWriter, r *http.Request) {
+	if err := h.rules.Delete(r.Context(), r.PathValue("id")); err != nil {
+		h.writeStoreErr(w, err, "deleting rule")
+		return
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
+func (h *Handler) handleListDeliveries(w http.ResponseWriter, r *http.Request) {
+	entries, err := h.deliveries.ListForRule(r.Context(), r.PathValue("id"), 100)
+	if err != nil {
+		h.logger.Error("listing deliveries", "rule_id", r.PathValue("id"), "error", err)
+		writeError(w, http.StatusInternalServerError, "listing deliveries failed")
+		return
+	}
+	writeJSON(w, http.StatusOK, entries)
+}
+
+func (h *Handler) handleCreateTarget(w http.ResponseWriter, r *http.Request) {
+	var target notifystore.Target
+	if !decodeJSON(w, r, &target) {
+		return
+	}
+	if target.Name == "" {
+		writeError(w, http.StatusBadRequest, "name must not be empty")
+		return
+	}
+	if !notifystore.ValidKind(target.Kind) {
+		writeError(w, http.StatusBadRequest, "kind must be one of webhook, slack, pagerduty")
+		return
+	}
+	if target.WebhookURL == "" {
+		writeError(w, http.StatusBadRequest, "webhook_url must not be empty")
+		return
+	}
+	if err := h.targets.Create(r.Context(), &target); err != nil {
+		h.logger.Error("creating notification target", "error", err)
+		writeError(w, http.StatusInternalServerError, "creating notification target failed")
+		return
+	}
+	writeJSON(w, http.StatusCreated, target)
+}
+
+func (h *Handler) handleListTargets(w http.ResponseWriter, r *http.Request) {
+	targets, err := h.targets.List(r.Context())
+	if err != nil {
+		h.logger.Error("listing notification targets", "error", err)
+		writeError(w, http.StatusInternalServerError, "listing notification targets failed")
+		return
+	}
+	writeJSON(w, http.StatusOK, targets)
+}
+
+func (h *Handler) handleGetTarget(w http.ResponseWriter, r *http.Request) {
+	target, err := h.targets.Get(r.Context(), r.PathValue("id"))
+	if err != nil {
+		h.writeStoreErr(w, err, "fetching notification target")
+		return
+	}
+	writeJSON(w, http.StatusOK, target)
+}
+
+func (h *Handler) handleDeleteTarget(w http.ResponseWriter, r *http.Request) {
+	if err := h.targets.Delete(r.Context(), r.PathValue("id")); err != nil {
+		h.writeStoreErr(w, err, "deleting notification target")
+		return
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
+// validateRule enforces the shape the evaluator assumes:
+// condition_type-appropriate fields present, a sane interval floor.
+// Mirrors the DB CHECK constraints so bad input is rejected with a
+// clear message here rather than surfacing as an opaque constraint
+// violation from Postgres.
+func validateRule(r *rulestore.Rule) error {
+	if r.Name == "" {
+		return errBadRequest("name must not be empty")
+	}
+	if r.Query == "" {
+		return errBadRequest("query must not be empty")
+	}
+	if r.NotificationTargetID == "" {
+		return errBadRequest("notification_target_id must not be empty")
+	}
+	if r.EvalIntervalSeconds < 30 {
+		return errBadRequest("eval_interval_seconds must be at least 30")
+	}
+	switch r.ConditionType {
+	case rulestore.ConditionThreshold:
+		if r.Comparator == nil || !rulestore.ValidComparator(*r.Comparator) {
+			return errBadRequest("threshold rules require a valid comparator (gt, gte, lt, lte, eq, ne)")
+		}
+		if r.ThresholdValue == nil {
+			return errBadRequest("threshold rules require threshold_value")
+		}
+	case rulestore.ConditionAbsence:
+		// No comparator/threshold_value needed -- absence is "the query
+		// returned zero rows in its own earliest=/latest= window."
+	default:
+		return errBadRequest("condition_type must be \"threshold\" or \"absence\"")
+	}
+	return nil
+}
+
+type badRequestError string
+
+func (e badRequestError) Error() string { return string(e) }
+func errBadRequest(msg string) error    { return badRequestError(msg) }
+
+func (h *Handler) writeStoreErr(w http.ResponseWriter, err error, action string) {
+	if errors.Is(err, rulestore.ErrNotFound) || errors.Is(err, notifystore.ErrNotFound) {
+		writeError(w, http.StatusNotFound, "not found")
+		return
+	}
+	h.logger.Error(action, "error", err)
+	writeError(w, http.StatusInternalServerError, action+" failed")
+}
+
+func decodeJSON(w http.ResponseWriter, r *http.Request, v any) bool {
+	r.Body = http.MaxBytesReader(w, r.Body, maxBodyBytes)
+	if err := json.NewDecoder(r.Body).Decode(v); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid JSON body: "+err.Error())
+		return false
+	}
+	return true
+}
+
+func writeJSON(w http.ResponseWriter, status int, v any) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	_ = json.NewEncoder(w).Encode(v)
+}
+
+type errorResponse struct {
+	Error string `json:"error"`
+}
+
+func writeError(w http.ResponseWriter, status int, msg string) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	_ = json.NewEncoder(w).Encode(errorResponse{Error: msg})
+}
@@ -0,0 +1,261 @@
+package httpapi
+
+import (
+	"context"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/sentry/sentry/alerting/internal/notifystore"
+	"github.com/sentry/sentry/alerting/internal/rulestore"
+)
+
+type fakeRuleStore struct {
+	rules map[string]*rulestore.RuleWithState
+}
+
+func newFakeRuleStore() *fakeRuleStore {
+	return &fakeRuleStore{rules: map[string]*rulestore.RuleWithState{}}
+}
+
+func (f *fakeRuleStore) Create(_ context.Context, r *rulestore.Rule) error {
+	r.ID = "rule-1"
+	f.rules[r.ID] = &rulestore.RuleWithState{Rule: *r, State: rulestore.AlertState{RuleID: r.ID, State: rulestore.StateOK}}
+	return nil
+}
+
+func (f *fakeRuleStore) List(_ context.Context) ([]rulestore.RuleWithState, error) {
+	var out []rulestore.RuleWithState
+	for _, r := range f.rules {
+		out = append(out, *r)
+	}
+	return out, nil
+}
+
+func (f *fakeRuleStore) Get(_ context.Context, id string) (*rulestore.RuleWithState, error) {
+	r, ok := f.rules[id]
+	if !ok {
+		return nil, rulestore.ErrNotFound
+	}
+	return r, nil
+}
+
+func (f *fakeRuleStore) Delete(_ context.Context, id string) error {
+	if _, ok := f.rules[id]; !ok {
+		return rulestore.ErrNotFound
+	}
+	delete(f.rules, id)
+	return nil
+}
+
+type fakeTargetStore struct {
+	targets map[string]*notifystore.Target
+}
+
+func newFakeTargetStore() *fakeTargetStore {
+	return &fakeTargetStore{targets: map[string]*notifystore.Target{}}
+}
+
+func (f *fakeTargetStore) Create(_ context.Context, t *notifystore.Target) error {
+	t.ID = "target-1"
+	f.targets[t.ID] = t
+	return nil
+}
+func (f *fakeTargetStore) List(_ context.Context) ([]notifystore.Target, error) {
+	var out []notifystore.Target
+	for _, t := range f.targets {
+		out = append(out, *t)
+	}
+	return out, nil
+}
+func (f *fakeTargetStore) Get(_ context.Context, id string) (*notifystore.Target, error) {
+	t, ok := f.targets[id]
+	if !ok {
+		return nil, notifystore.ErrNotFound
+	}
+	return t, nil
+}
+func (f *fakeTargetStore) Delete(_ context.Context, id string) error {
+	if _, ok := f.targets[id]; !ok {
+		return notifystore.ErrNotFound
+	}
+	delete(f.targets, id)
+	return nil
+}
+
+type fakeDeliveryReader struct {
+	entries []rulestore.DeliveryLogEntry
+}
+
+func (f *fakeDeliveryReader) ListForRule(_ context.Context, _ string, _ int) ([]rulestore.DeliveryLogEntry, error) {
+	return f.entries, nil
+}
+
+func newTestMux(rules ruleStore, targets targetStore, deliveries deliveryReader) *http.ServeMux {
+	h := NewHandler(slog.New(slog.NewTextHandler(io.Discard, nil)), rules, targets, deliveries)
+	mux := http.NewServeMux()
+	h.RegisterRoutes(mux)
+	return mux
+}
+
+func doRequest(t *testing.T, mux *http.ServeMux, method, path, body string) *httptest.ResponseRecorder {
+	t.Helper()
+	var r io.Reader
+	if body != "" {
+		r = strings.NewReader(body)
+	}
+	req := httptest.NewRequest(method, path, r)
+	rec := httptest.NewRecorder()
+	mux.ServeHTTP(rec, req)
+	return rec
+}
+
+func TestCreateThresholdRule(t *testing.T) {
+	targets := newFakeTargetStore()
+	targets.targets["target-1"] = &notifystore.Target{ID: "target-1"}
+	mux := newTestMux(newFakeRuleStore(), targets, &fakeDeliveryReader{})
+
+	rec := doRequest(t, mux, http.MethodPost, "/rules", `{
+		"name": "High error rate", "query": "service=api | where status>=500 | stats count",
+		"condition_type": "threshold", "comparator": "gt", "threshold_value": 100,
+		"eval_interval_seconds": 60, "notification_target_id": "target-1"
+	}`)
+	if rec.Code != http.StatusCreated {
+		t.Fatalf("status = %d, want 201; body=%s", rec.Code, rec.Body.String())
+	}
+}
+
+// TestCreateRuleDefaultsToEnabledWhenOmitted guards against a real bug
+// caught by actually calling this endpoint: a plain `bool` JSON field
+// can't distinguish "omitted" from "explicitly false," and Go's zero
+// value for bool is false -- without createRuleRequest's *bool handling,
+// a create request that simply didn't mention "enabled" silently created
+// a rule the evaluator's claim query would never pick up.
+func TestCreateRuleDefaultsToEnabledWhenOmitted(t *testing.T) {
+	rules := newFakeRuleStore()
+	mux := newTestMux(rules, newFakeTargetStore(), &fakeDeliveryReader{})
+
+	rec := doRequest(t, mux, http.MethodPost, "/rules", `{
+		"name": "no enabled field", "query": "service=api", "condition_type": "absence",
+		"eval_interval_seconds": 60, "notification_target_id": "target-1"
+	}`)
+	if rec.Code != http.StatusCreated {
+		t.Fatalf("status = %d, want 201; body=%s", rec.Code, rec.Body.String())
+	}
+	if !rules.rules["rule-1"].Enabled {
+		t.Fatalf("expected a rule created without an explicit \"enabled\" field to default to enabled=true")
+	}
+}
+
+func TestCreateRuleRespectsExplicitDisabled(t *testing.T) {
+	rules := newFakeRuleStore()
+	mux := newTestMux(rules, newFakeTargetStore(), &fakeDeliveryReader{})
+
+	rec := doRequest(t, mux, http.MethodPost, "/rules", `{
+		"name": "explicitly disabled", "query": "service=api", "condition_type": "absence",
+		"eval_interval_seconds": 60, "notification_target_id": "target-1", "enabled": false
+	}`)
+	if rec.Code != http.StatusCreated {
+		t.Fatalf("status = %d, want 201; body=%s", rec.Code, rec.Body.String())
+	}
+	if rules.rules["rule-1"].Enabled {
+		t.Fatalf("expected an explicit \"enabled\": false to be respected")
+	}
+}
+
+func TestCreateThresholdRuleRejectsMissingComparator(t *testing.T) {
+	mux := newTestMux(newFakeRuleStore(), newFakeTargetStore(), &fakeDeliveryReader{})
+	rec := doRequest(t, mux, http.MethodPost, "/rules", `{
+		"name": "bad rule", "query": "service=api", "condition_type": "threshold",
+		"eval_interval_seconds": 60, "notification_target_id": "target-1"
+	}`)
+	if rec.Code != http.StatusBadRequest {
+		t.Fatalf("status = %d, want 400; body=%s", rec.Code, rec.Body.String())
+	}
+}
+
+func TestCreateAbsenceRuleDoesNotRequireComparator(t *testing.T) {
+	mux := newTestMux(newFakeRuleStore(), newFakeTargetStore(), &fakeDeliveryReader{})
+	rec := doRequest(t, mux, http.MethodPost, "/rules", `{
+		"name": "no heartbeat", "query": "service=payments earliest=-5m", "condition_type": "absence",
+		"eval_interval_seconds": 60, "notification_target_id": "target-1"
+	}`)
+	if rec.Code != http.StatusCreated {
+		t.Fatalf("status = %d, want 201; body=%s", rec.Code, rec.Body.String())
+	}
+}
+
+func TestCreateRuleRejectsShortInterval(t *testing.T) {
+	mux := newTestMux(newFakeRuleStore(), newFakeTargetStore(), &fakeDeliveryReader{})
+	rec := doRequest(t, mux, http.MethodPost, "/rules", `{
+		"name": "too fast", "query": "service=api", "condition_type": "absence",
+		"eval_interval_seconds": 5, "notification_target_id": "target-1"
+	}`)
+	if rec.Code != http.StatusBadRequest {
+		t.Fatalf("status = %d, want 400; body=%s", rec.Code, rec.Body.String())
+	}
+}
+
+func TestGetRuleNotFound(t *testing.T) {
+	mux := newTestMux(newFakeRuleStore(), newFakeTargetStore(), &fakeDeliveryReader{})
+	rec := doRequest(t, mux, http.MethodGet, "/rules/nope", "")
+	if rec.Code != http.StatusNotFound {
+		t.Fatalf("status = %d, want 404", rec.Code)
+	}
+}
+
+func TestDeleteRule(t *testing.T) {
+	rules := newFakeRuleStore()
+	rules.rules["rule-1"] = &rulestore.RuleWithState{Rule: rulestore.Rule{ID: "rule-1"}}
+	mux := newTestMux(rules, newFakeTargetStore(), &fakeDeliveryReader{})
+
+	rec := doRequest(t, mux, http.MethodDelete, "/rules/rule-1", "")
+	if rec.Code != http.StatusNoContent {
+		t.Fatalf("status = %d, want 204", rec.Code)
+	}
+	if _, ok := rules.rules["rule-1"]; ok {
+		t.Fatalf("expected rule to be deleted")
+	}
+}
+
+func TestCreateTargetRejectsInvalidKind(t *testing.T) {
+	mux := newTestMux(newFakeRuleStore(), newFakeTargetStore(), &fakeDeliveryReader{})
+	rec := doRequest(t, mux, http.MethodPost, "/targets", `{"name": "x", "kind": "carrier-pigeon", "webhook_url": "https://example.com"}`)
+	if rec.Code != http.StatusBadRequest {
+		t.Fatalf("status = %d, want 400; body=%s", rec.Code, rec.Body.String())
+	}
+}
+
+func TestCreateSlackTarget(t *testing.T) {
+	mux := newTestMux(newFakeRuleStore(), newFakeTargetStore(), &fakeDeliveryReader{})
+	rec := doRequest(t, mux, http.MethodPost, "/targets", `{"name": "oncall", "kind": "slack", "webhook_url": "https://hooks.slack.com/services/x"}`)
+	if rec.Code != http.StatusCreated {
+		t.Fatalf("status = %d, want 201; body=%s", rec.Code, rec.Body.String())
+	}
+}
+
+func TestListDeliveriesForRule(t *testing.T) {
+	deliveries := &fakeDeliveryReader{entries: []rulestore.DeliveryLogEntry{
+		{ID: 1, RuleID: "rule-1", EventType: "firing", Status: "sent"},
+	}}
+	mux := newTestMux(newFakeRuleStore(), newFakeTargetStore(), deliveries)
+
+	rec := doRequest(t, mux, http.MethodGet, "/rules/rule-1/deliveries", "")
+	if rec.Code != http.StatusOK {
+		t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
+	}
+	if !strings.Contains(rec.Body.String(), `"status":"sent"`) {
+		t.Fatalf("expected delivery entry in response, got: %s", rec.Body.String())
+	}
+}
+
+func TestHandleHealthz(t *testing.T) {
+	mux := newTestMux(newFakeRuleStore(), newFakeTargetStore(), &fakeDeliveryReader{})
+	rec := doRequest(t, mux, http.MethodGet, "/healthz", "")
+	if rec.Code != http.StatusOK {
+		t.Fatalf("status = %d, want 200", rec.Code)
+	}
+}