Phase 3: dashboards and alerting

Saved, shareable multi-panel dashboards (table/line/bar/single-stat
panels via gridstack + uPlot, global + per-panel time range, JSON
export/import) and threshold/absence alert rules with an
ok/pending/firing evaluator and webhook/Slack/PagerDuty delivery.

- New /metadata component: Postgres control-plane store for dashboards,
  panels, notification targets, alert rules/state, and delivery log --
  see docs/phase-3-dashboard-design.md for why ClickHouse's MergeTree
  family isn't a fit for this access pattern (needs real row-level
  locking and read-your-writes consistency).
- api/internal/dashboards: dashboard/panel CRUD, pure -- panel query
  execution stays client-side, reusing the existing /query endpoint.
- New /alerting service: rule/target CRUD, a ticker-driven evaluator
  (claim-then-evaluate concurrency control, transactional-outbox
  delivery, query errors and threshold zero-rows never coerced into a
  false transition) and webhook/Slack/PagerDuty delivery with
  retry/backoff. See docs/phase-3-alerting-design.md for the full
  state-machine design and the four correctness properties it
  implements.
- web: /dashboards and /alerts UIs; cli: sentryctl dashboards/alerts
  list/get/apply, seeding a future Terraform provider's JSON contract.
- hack/alert-load-test: 500 rules against real ClickHouse data, real
  measured results in docs/phase-3-runbook.md.

Five real bugs found by actually running this against a live stack
(documented in the runbook, not just fixed silently): a latent Phase 2
bug where ClickHouse rejected the timestamp format used for
earliest=/latest= queries; a "now" literal token injected into query
text; a GridStack/uPlot layout-timing race; JS's Date.parse being too
lenient to use as a timestamp-detection heuristic; a rule's "enabled"
field silently defaulting to false when omitted; and the evaluator's
claim-batch-size and worker-pool-concurrency defaulting to the same
value, causing 500 concurrently-due rules to take 125s to cycle through
instead of the configured 60s.

api/cmd/api/main.go

@@ -7,18 +7,23 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"log/slog"
 	"net/http"
 	"os"
 	"os/signal"
+	"strings"
 	"syscall"
 	"time"
 
 	"github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/jackc/pgx/v5/pgxpool"
 
 	"github.com/sentry/sentry/api/internal/config"
-	"github.com/sentry/sentry/api/internal/querylang/executor"
+	"github.com/sentry/sentry/api/internal/dashboards"
+	"github.com/sentry/sentry/api/internal/httpserver"
 	"github.com/sentry/sentry/api/internal/queryapi"
+	"github.com/sentry/sentry/api/internal/querylang/executor"
 	"github.com/sentry/sentry/api/internal/searchclient"
 )
 
@@ -31,6 +36,16 @@ func main() {
 		os.Exit(1)
 	}
 
+	// -healthcheck: a self-check mode for Docker's HEALTHCHECK, not a
+	// flag anyone runs by hand. The api image is distroless (no shell,
+	// no wget/curl -- see api/Dockerfile), so docker-compose's
+	// healthcheck execs this binary against itself instead of an
+	// external tool. Exits before any ClickHouse/Postgres/search dial,
+	// since those aren't what "is the HTTP server up" is asking.
+	if len(os.Args) > 1 && os.Args[1] == "-healthcheck" {
+		os.Exit(runHealthcheck(cfg.HTTPListenAddr))
+	}
+
 	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
 	defer stop()
 
@@ -60,12 +75,33 @@ func main() {
 	}
 	defer search.Close()
 
+	pgDSN := fmt.Sprintf("postgres://%s:%s@%s/%s", cfg.Postgres.Username, cfg.Postgres.Password, cfg.Postgres.Addr, cfg.Postgres.Database)
+	pgPool, err := pgxpool.New(ctx, pgDSN)
+	if err != nil {
+		logger.Error("opening postgres pool", "error", err)
+		os.Exit(1)
+	}
+	defer pgPool.Close()
+
+	if err := pgPool.Ping(ctx); err != nil {
+		logger.Error("pinging postgres", "error", err)
+		os.Exit(1)
+	}
+
 	sqlRunner := executor.NewChRunner(conn)
-	handler := queryapi.NewHandler(logger, sqlRunner, search, cfg.QueryTimeout, cfg.CORSAllowedOrigin)
+	queryHandler := queryapi.NewHandler(logger, sqlRunner, search, cfg.QueryTimeout)
+	dashboardsHandler := dashboards.NewHandler(logger, dashboards.NewStore(pgPool))
+
+	// One shared mux, CORS applied once around the whole thing -- see
+	// internal/httpserver's doc comment for why this changed from each
+	// handler wrapping itself individually.
+	mux := http.NewServeMux()
+	queryHandler.RegisterRoutes(mux)
+	dashboardsHandler.RegisterRoutes(mux)
 
 	srv := &http.Server{
 		Addr:    cfg.HTTPListenAddr,
-		Handler: handler.Routes(),
+		Handler: httpserver.WithCORS(mux, cfg.CORSAllowedOrigin),
 	}
 
 	errCh := make(chan error, 1)
@@ -88,3 +124,25 @@ func main() {
 		}
 	}
 }
+
+// runHealthcheck GETs its own /healthz and returns an exit code, for
+// Docker's HEALTHCHECK to exec directly (see the -healthcheck flag
+// above). listenAddr is HTTP_LISTEN_ADDR-shaped (e.g. ":8080") --
+// "localhost" replaces a bare host part since that's this same
+// container reaching itself, not another service.
+func runHealthcheck(listenAddr string) int {
+	addr := listenAddr
+	if strings.HasPrefix(addr, ":") {
+		addr = "localhost" + addr
+	}
+	client := http.Client{Timeout: 3 * time.Second}
+	resp, err := client.Get("http://" + addr + "/healthz")
+	if err != nil {
+		return 1
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return 1
+	}
+	return 0
+}