Phase 3: dashboards and alerting
Saved, shareable multi-panel dashboards (table/line/bar/single-stat panels via gridstack + uPlot, global + per-panel time range, JSON export/import) and threshold/absence alert rules with an ok/pending/firing evaluator and webhook/Slack/PagerDuty delivery. - New /metadata component: Postgres control-plane store for dashboards, panels, notification targets, alert rules/state, and delivery log -- see docs/phase-3-dashboard-design.md for why ClickHouse's MergeTree family isn't a fit for this access pattern (needs real row-level locking and read-your-writes consistency). - api/internal/dashboards: dashboard/panel CRUD, pure -- panel query execution stays client-side, reusing the existing /query endpoint. - New /alerting service: rule/target CRUD, a ticker-driven evaluator (claim-then-evaluate concurrency control, transactional-outbox delivery, query errors and threshold zero-rows never coerced into a false transition) and webhook/Slack/PagerDuty delivery with retry/backoff. See docs/phase-3-alerting-design.md for the full state-machine design and the four correctness properties it implements. - web: /dashboards and /alerts UIs; cli: sentryctl dashboards/alerts list/get/apply, seeding a future Terraform provider's JSON contract. - hack/alert-load-test: 500 rules against real ClickHouse data, real measured results in docs/phase-3-runbook.md. Five real bugs found by actually running this against a live stack (documented in the runbook, not just fixed silently): a latent Phase 2 bug where ClickHouse rejected the timestamp format used for earliest=/latest= queries; a "now" literal token injected into query text; a GridStack/uPlot layout-timing race; JS's Date.parse being too lenient to use as a timestamp-detection heuristic; a rule's "enabled" field silently defaulting to false when omitted; and the evaluator's claim-batch-size and worker-pool-concurrency defaulting to the same value, causing 500 concurrently-due rules to take 125s to cycle through instead of the configured 60s.
This commit is contained in:
@@ -0,0 +1,88 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/sentry/sentry/alerting/internal/notifystore"
|
||||
)
|
||||
|
||||
func TestBuildPayloadGenericDefaultShape(t *testing.T) {
|
||||
target := notifystore.Target{Kind: notifystore.KindWebhook}
|
||||
value := 142.0
|
||||
payload, err := BuildPayload(target, Event{
|
||||
RuleID: "r1", RuleName: "High error rate", EventType: "firing",
|
||||
Value: &value, Timestamp: time.Date(2026, 8, 14, 0, 0, 0, 0, time.UTC),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("BuildPayload: %v", err)
|
||||
}
|
||||
var got defaultGenericPayload
|
||||
if err := json.Unmarshal(payload, &got); err != nil {
|
||||
t.Fatalf("unmarshaling payload: %v", err)
|
||||
}
|
||||
if got.RuleName != "High error rate" || got.EventType != "firing" || got.Value == nil || *got.Value != 142.0 {
|
||||
t.Fatalf("unexpected payload: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPayloadGenericUsesTemplate(t *testing.T) {
|
||||
tmpl := `{"custom": "{{.RuleName}} is {{.EventType}}"}`
|
||||
target := notifystore.Target{Kind: notifystore.KindWebhook, PayloadTemplate: &tmpl}
|
||||
payload, err := BuildPayload(target, Event{RuleName: "disk full", EventType: "firing"})
|
||||
if err != nil {
|
||||
t.Fatalf("BuildPayload: %v", err)
|
||||
}
|
||||
var got map[string]string
|
||||
if err := json.Unmarshal(payload, &got); err != nil {
|
||||
t.Fatalf("unmarshaling templated payload: %v", err)
|
||||
}
|
||||
if got["custom"] != "disk full is firing" {
|
||||
t.Fatalf("got %q", got["custom"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPayloadSlackShape(t *testing.T) {
|
||||
target := notifystore.Target{Kind: notifystore.KindSlack}
|
||||
payload, err := BuildPayload(target, Event{RuleName: "High error rate", EventType: "resolved"})
|
||||
if err != nil {
|
||||
t.Fatalf("BuildPayload: %v", err)
|
||||
}
|
||||
var got slackPayload
|
||||
if err := json.Unmarshal(payload, &got); err != nil {
|
||||
t.Fatalf("unmarshaling slack payload: %v", err)
|
||||
}
|
||||
if got.Text == "" {
|
||||
t.Fatalf("expected non-empty Slack text")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPayloadPagerDutyMapsEventActionAndRoutingKey(t *testing.T) {
|
||||
secret := "R0UT1NGKEY"
|
||||
target := notifystore.Target{Kind: notifystore.KindPagerDuty, Secret: &secret}
|
||||
|
||||
firing, err := BuildPayload(target, Event{RuleID: "r1", RuleName: "disk full", EventType: "firing"})
|
||||
if err != nil {
|
||||
t.Fatalf("BuildPayload firing: %v", err)
|
||||
}
|
||||
var gotFiring pagerDutyPayload
|
||||
if err := json.Unmarshal(firing, &gotFiring); err != nil {
|
||||
t.Fatalf("unmarshaling: %v", err)
|
||||
}
|
||||
if gotFiring.EventAction != "trigger" || gotFiring.RoutingKey != secret || gotFiring.DedupKey != "r1" {
|
||||
t.Fatalf("unexpected firing payload: %+v", gotFiring)
|
||||
}
|
||||
|
||||
resolved, err := BuildPayload(target, Event{RuleID: "r1", RuleName: "disk full", EventType: "resolved"})
|
||||
if err != nil {
|
||||
t.Fatalf("BuildPayload resolved: %v", err)
|
||||
}
|
||||
var gotResolved pagerDutyPayload
|
||||
if err := json.Unmarshal(resolved, &gotResolved); err != nil {
|
||||
t.Fatalf("unmarshaling: %v", err)
|
||||
}
|
||||
if gotResolved.EventAction != "resolve" {
|
||||
t.Fatalf("expected event_action=resolve, got %q", gotResolved.EventAction)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/sentry/sentry/alerting/internal/notifystore"
|
||||
)
|
||||
|
||||
// pagerDutyPayload is PagerDuty's Events API v2 shape. routing_key comes
|
||||
// from the target's `secret` field -- PagerDuty calls this an
|
||||
// "integration key," not a delivery credential in the auth sense, but
|
||||
// it's stored in the same plaintext `secret` column as any other
|
||||
// target's credential (see /docs/phase-3-alerting-design.md's "Known
|
||||
// gaps").
|
||||
type pagerDutyPayload struct {
|
||||
RoutingKey string `json:"routing_key"`
|
||||
EventAction string `json:"event_action"` // "trigger" | "resolve"
|
||||
DedupKey string `json:"dedup_key"` // rule_id -- lets PagerDuty correlate trigger/resolve as one incident
|
||||
Payload pagerDutyEventPayload `json:"payload"`
|
||||
}
|
||||
|
||||
type pagerDutyEventPayload struct {
|
||||
Summary string `json:"summary"`
|
||||
Source string `json:"source"`
|
||||
Severity string `json:"severity"`
|
||||
}
|
||||
|
||||
func buildPagerDutyPayload(target notifystore.Target, event Event) ([]byte, error) {
|
||||
action := "trigger"
|
||||
severity := "critical"
|
||||
if event.EventType == "resolved" {
|
||||
action = "resolve"
|
||||
severity = "info"
|
||||
}
|
||||
|
||||
var routingKey string
|
||||
if target.Secret != nil {
|
||||
routingKey = *target.Secret
|
||||
}
|
||||
|
||||
summary := fmt.Sprintf("%s: %s", event.RuleName, event.EventType)
|
||||
if event.Value != nil && event.ThresholdValue != nil {
|
||||
summary = fmt.Sprintf("%s (value %.4g %s %.4g)", summary, *event.Value, comparatorSymbol(event.Comparator), *event.ThresholdValue)
|
||||
}
|
||||
|
||||
return json.Marshal(pagerDutyPayload{
|
||||
RoutingKey: routingKey,
|
||||
EventAction: action,
|
||||
DedupKey: event.RuleID,
|
||||
Payload: pagerDutyEventPayload{
|
||||
Summary: summary,
|
||||
Source: "sentry",
|
||||
Severity: severity,
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// slackPayload is Slack's incoming-webhook shape: {"text": "..."}. No
|
||||
// user template for this kind -- see webhook.go's BuildPayload doc
|
||||
// comment for why (thin formatter, not a user-configurable path).
|
||||
type slackPayload struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
func buildSlackPayload(event Event) ([]byte, error) {
|
||||
var text string
|
||||
switch event.EventType {
|
||||
case "firing":
|
||||
text = fmt.Sprintf(":rotating_light: *%s* is firing", event.RuleName)
|
||||
if event.Value != nil && event.ThresholdValue != nil {
|
||||
text += fmt.Sprintf(" (value %.4g %s %.4g)", *event.Value, comparatorSymbol(event.Comparator), *event.ThresholdValue)
|
||||
}
|
||||
case "resolved":
|
||||
text = fmt.Sprintf(":white_check_mark: *%s* resolved", event.RuleName)
|
||||
default:
|
||||
text = fmt.Sprintf("%s: %s", event.RuleName, event.EventType)
|
||||
}
|
||||
return json.Marshal(slackPayload{Text: text})
|
||||
}
|
||||
|
||||
func comparatorSymbol(c string) string {
|
||||
switch c {
|
||||
case "gt":
|
||||
return ">"
|
||||
case "gte":
|
||||
return ">="
|
||||
case "lt":
|
||||
return "<"
|
||||
case "lte":
|
||||
return "<="
|
||||
case "eq":
|
||||
return "=="
|
||||
case "ne":
|
||||
return "!="
|
||||
default:
|
||||
return c
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,229 @@
|
||||
// Package delivery sends notifications for firing/resolved alert
|
||||
// events. All three notification_targets.kind values -- webhook, slack,
|
||||
// pagerduty -- go through the exact same claim-then-POST-with-backoff
|
||||
// mechanism in this file; slack.go and pagerduty.go are payload
|
||||
// *formatters* only, never a separate delivery path, per
|
||||
// /docs/phase-3-alerting-design.md's "thin wrappers over the generic
|
||||
// webhook" requirement.
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"github.com/sentry/sentry/alerting/internal/notifystore"
|
||||
)
|
||||
|
||||
// Event is the firing/resolved occurrence a payload is rendered from.
|
||||
type Event struct {
|
||||
RuleID string
|
||||
RuleName string
|
||||
EventType string // "firing" | "resolved"
|
||||
ConditionType string
|
||||
Comparator string
|
||||
ThresholdValue *float64
|
||||
Value *float64
|
||||
Timestamp time.Time
|
||||
}
|
||||
|
||||
// defaultGenericPayload is used when a webhook target has no
|
||||
// payload_template.
|
||||
type defaultGenericPayload struct {
|
||||
RuleID string `json:"rule_id"`
|
||||
RuleName string `json:"rule_name"`
|
||||
EventType string `json:"event_type"`
|
||||
Value *float64 `json:"value,omitempty"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
}
|
||||
|
||||
// BuildPayload dispatches by target.Kind to the right formatter. This is
|
||||
// the only place kind-specific formatting logic lives -- everything
|
||||
// downstream of this (worker.go's send loop) is kind-agnostic.
|
||||
func BuildPayload(target notifystore.Target, event Event) ([]byte, error) {
|
||||
switch target.Kind {
|
||||
case notifystore.KindSlack:
|
||||
return buildSlackPayload(event)
|
||||
case notifystore.KindPagerDuty:
|
||||
return buildPagerDutyPayload(target, event)
|
||||
default: // webhook
|
||||
return buildGenericPayload(target, event)
|
||||
}
|
||||
}
|
||||
|
||||
func buildGenericPayload(target notifystore.Target, event Event) ([]byte, error) {
|
||||
if target.PayloadTemplate == nil || *target.PayloadTemplate == "" {
|
||||
return json.Marshal(defaultGenericPayload{
|
||||
RuleID: event.RuleID, RuleName: event.RuleName, EventType: event.EventType,
|
||||
Value: event.Value, Timestamp: event.Timestamp.UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
tmpl, err := template.New("payload").Parse(*target.PayloadTemplate)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing payload_template: %w", err)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := tmpl.Execute(&buf, event); err != nil {
|
||||
return nil, fmt.Errorf("rendering payload_template: %w", err)
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// Worker claims pending/retrying delivery_log rows and sends them.
|
||||
// Deliberately separate from the evaluator: "decided to notify"
|
||||
// (rulestore.ApplyTransition's transactional outbox insert) is
|
||||
// transactionally certain; "the HTTP call succeeded" is best-effort with
|
||||
// retries, handled entirely here.
|
||||
type Worker struct {
|
||||
pool *pgxpool.Pool
|
||||
notifications *notifystore.Store
|
||||
http *http.Client
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
func NewWorker(pool *pgxpool.Pool, notifications *notifystore.Store, logger *slog.Logger) *Worker {
|
||||
return &Worker{pool: pool, notifications: notifications, http: &http.Client{Timeout: 10 * time.Second}, logger: logger}
|
||||
}
|
||||
|
||||
// Run claims and sends due deliveries every tickInterval until ctx is
|
||||
// cancelled.
|
||||
func (w *Worker) Run(ctx context.Context, tickInterval time.Duration) error {
|
||||
ticker := time.NewTicker(tickInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-ticker.C:
|
||||
w.processDue(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type claimedDelivery struct {
|
||||
id int64
|
||||
ruleID string
|
||||
notificationTargetID string
|
||||
eventType string
|
||||
attemptCount int
|
||||
maxAttempts int
|
||||
payload []byte
|
||||
}
|
||||
|
||||
// processDue claims due rows (same SKIP LOCKED pattern as the
|
||||
// evaluator's rule claim -- see rulestore.ClaimDueRules) and attempts
|
||||
// delivery for each. One claim batch per tick; a stuck/slow target
|
||||
// doesn't block others since each send happens independently.
|
||||
func (w *Worker) processDue(ctx context.Context) {
|
||||
rows, err := w.pool.Query(ctx, `
|
||||
WITH due AS (
|
||||
SELECT id FROM delivery_log
|
||||
WHERE status IN ('pending', 'retrying') AND (next_attempt_at IS NULL OR next_attempt_at <= now())
|
||||
ORDER BY created_at
|
||||
LIMIT 50
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE delivery_log d
|
||||
SET last_attempt_at = now()
|
||||
FROM due
|
||||
WHERE d.id = due.id
|
||||
RETURNING d.id, d.rule_id, d.notification_target_id, d.event_type, d.attempt_count, d.max_attempts, d.payload`)
|
||||
if err != nil {
|
||||
w.logger.Error("claiming due deliveries", "error", err)
|
||||
return
|
||||
}
|
||||
var claimed []claimedDelivery
|
||||
for rows.Next() {
|
||||
var c claimedDelivery
|
||||
if err := rows.Scan(&c.id, &c.ruleID, &c.notificationTargetID, &c.eventType, &c.attemptCount, &c.maxAttempts, &c.payload); err != nil {
|
||||
w.logger.Error("scanning claimed delivery", "error", err)
|
||||
continue
|
||||
}
|
||||
claimed = append(claimed, c)
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
for _, c := range claimed {
|
||||
w.attempt(ctx, c)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) attempt(ctx context.Context, c claimedDelivery) {
|
||||
target, err := w.notifications.Get(ctx, c.notificationTargetID)
|
||||
if err != nil {
|
||||
w.fail(ctx, c, 0, fmt.Sprintf("looking up notification target: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, target.WebhookURL, bytes.NewReader(c.payload))
|
||||
if err != nil {
|
||||
w.fail(ctx, c, 0, fmt.Sprintf("building request: %v", err))
|
||||
return
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := w.http.Do(req)
|
||||
if err != nil {
|
||||
w.fail(ctx, c, 0, err.Error())
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
||||
w.markSent(ctx, c.id, resp.StatusCode)
|
||||
return
|
||||
}
|
||||
w.fail(ctx, c, resp.StatusCode, fmt.Sprintf("non-2xx response: %d", resp.StatusCode))
|
||||
}
|
||||
|
||||
func (w *Worker) markSent(ctx context.Context, id int64, statusCode int) {
|
||||
_, err := w.pool.Exec(ctx, `
|
||||
UPDATE delivery_log SET status = 'sent', attempt_count = attempt_count + 1, response_status = $1 WHERE id = $2`,
|
||||
statusCode, id)
|
||||
if err != nil {
|
||||
w.logger.Error("marking delivery sent", "delivery_id", id, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// fail records a failed attempt. If attempts remain, it's rescheduled
|
||||
// with exponential backoff (2^attempt_count seconds, capped at 1 hour);
|
||||
// otherwise it's marked permanently failed.
|
||||
func (w *Worker) fail(ctx context.Context, c claimedDelivery, statusCode int, errMsg string) {
|
||||
nextAttempt := c.attemptCount + 1
|
||||
if nextAttempt >= c.maxAttempts {
|
||||
_, err := w.pool.Exec(ctx, `
|
||||
UPDATE delivery_log SET status = 'failed', attempt_count = attempt_count + 1, response_status = $1, last_error = $2 WHERE id = $3`,
|
||||
nullIfZero(statusCode), errMsg, c.id)
|
||||
if err != nil {
|
||||
w.logger.Error("marking delivery failed", "delivery_id", c.id, "error", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
backoff := time.Duration(1<<uint(nextAttempt)) * time.Second
|
||||
if backoff > time.Hour {
|
||||
backoff = time.Hour
|
||||
}
|
||||
_, err := w.pool.Exec(ctx, `
|
||||
UPDATE delivery_log
|
||||
SET status = 'retrying', attempt_count = attempt_count + 1, response_status = $1, last_error = $2, next_attempt_at = now() + $3
|
||||
WHERE id = $4`,
|
||||
nullIfZero(statusCode), errMsg, backoff, c.id)
|
||||
if err != nil {
|
||||
w.logger.Error("scheduling delivery retry", "delivery_id", c.id, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func nullIfZero(n int) *int {
|
||||
if n == 0 {
|
||||
return nil
|
||||
}
|
||||
return &n
|
||||
}
|
||||
Reference in New Issue
Block a user