diff --git a/docker-compose.yml b/docker-compose.yml index 118bd18..ec124c9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,6 +43,7 @@ services: condition: service_healthy environment: REDPANDA_BROKERS: "redpanda:9092" + REDPANDA_ADMIN_HOSTS: "redpanda:9644" clickhouse: image: clickhouse/clickhouse-server:24.8 @@ -50,6 +51,17 @@ services: ports: - "8123:8123" # HTTP interface, used by the migrate step - "9000:9000" # native protocol, used by ingest and api + environment: + # The official image disables *network* access entirely for the + # default user (even from sibling containers) unless + # CLICKHOUSE_USER or CLICKHOUSE_PASSWORD is set to a genuinely + # non-empty value — confirmed by testing, not just reading docs: an + # explicitly-empty CLICKHOUSE_PASSWORD="" still triggers the + # lockdown, silently returning 403 to every other container. This + # password isn't a real secret (mTLS between agent and ingest is + # the actual security boundary here) — it exists purely to satisfy + # this image's login gate for local/homelab use. + CLICKHOUSE_PASSWORD: "sentry-dev-only" volumes: - clickhouse-data:/var/lib/clickhouse ulimits: @@ -73,6 +85,7 @@ services: condition: service_healthy environment: CLICKHOUSE_HTTP: "http://clickhouse:8123" + CLICKHOUSE_PASSWORD: "sentry-dev-only" ingest: build: @@ -89,6 +102,7 @@ services: environment: REDPANDA_BROKERS: "redpanda:9092" CLICKHOUSE_ADDR: "clickhouse:9000" + CLICKHOUSE_PASSWORD: "sentry-dev-only" # TLS_*_FILE env vars are left at their defaults # (/etc/sentry-ingest/{server,server-key,ca}.pem) — matches where # the volume below mounts the generated dev certs. @@ -107,6 +121,7 @@ services: - "8080:8080" environment: CLICKHOUSE_ADDR: "clickhouse:9000" + CLICKHOUSE_PASSWORD: "sentry-dev-only" web: build: diff --git a/storage/README.md b/storage/README.md index f7cc397..a05ff5b 100644 --- a/storage/README.md +++ b/storage/README.md @@ -59,18 +59,28 @@ split multi-statement files — keep each migration to a single statement. ## Running ```sh -docker compose up -d # starts a standalone ClickHouse for local work -./migrate.sh # applies migrations/*.sql +docker compose up -d # starts a standalone ClickHouse for local work +CLICKHOUSE_PASSWORD=sentry-dev-only ./migrate.sh # applies migrations/*.sql ``` -Environment variables `migrate.sh` reads (all optional, matching -`/ingest`'s ClickHouse defaults so the two stay in sync out of the box): +`CLICKHOUSE_PASSWORD` here has to match whatever `docker-compose.yml` set +on the `clickhouse` service — found this the hard way running the Phase 0 +runbook for real: the official ClickHouse image silently disables *all* +network access (including the published port, not just container-to- +container traffic) for the `default` user unless `CLICKHOUSE_USER` or +`CLICKHOUSE_PASSWORD` is a genuinely non-empty value. An empty +`CLICKHOUSE_PASSWORD=""` still triggers the lockdown — it has to actually +have a value. Not a real secret, just what this image demands. + +Environment variables `migrate.sh` reads (all optional except +`CLICKHOUSE_PASSWORD` as of the above, matching `/ingest`'s ClickHouse +defaults so the two stay in sync out of the box): | Var | Default | |---|---| | `CLICKHOUSE_HTTP` | `http://localhost:8123` | | `CLICKHOUSE_USER` | `default` | -| `CLICKHOUSE_PASSWORD` | (empty) | +| `CLICKHOUSE_PASSWORD` | (empty — override, see above) | | `CLICKHOUSE_DATABASE` | `sentry` | There's also a `Dockerfile` (bash + curl baked in, `migrations/` copied in diff --git a/storage/docker-compose.yml b/storage/docker-compose.yml index 2bfe576..bad33d6 100644 --- a/storage/docker-compose.yml +++ b/storage/docker-compose.yml @@ -9,6 +9,15 @@ services: ports: - "8123:8123" # HTTP interface, used by migrate.sh - "9000:9000" # native protocol, used by ingest + environment: + # The official image disables *network* access entirely for the + # default user (this includes the published port, not just + # container-to-container traffic) unless CLICKHOUSE_USER or + # CLICKHOUSE_PASSWORD is set to a genuinely non-empty value — an + # explicitly-empty CLICKHOUSE_PASSWORD="" still triggers it. Not a + # real secret; see the root docker-compose.yml for the full + # explanation. + CLICKHOUSE_PASSWORD: "sentry-dev-only" volumes: - clickhouse-data:/var/lib/clickhouse ulimits: diff --git a/transport/README.md b/transport/README.md index 29776fa..830fb15 100644 --- a/transport/README.md +++ b/transport/README.md @@ -15,9 +15,17 @@ both are invoked. ```sh docker compose up -d -REDPANDA_BROKERS=localhost:9092 ./provision-topics.sh +./provision-topics.sh # defaults (localhost:9092 / localhost:9644) match this compose file ``` +Two separate addresses matter here, confirmed by actually running this +against a live Redpanda container: `rpk cluster health` talks to the +**Admin API** (`REDPANDA_ADMIN_HOSTS`, port 9644), while `rpk topic ...` +talks to the **Kafka API** (`REDPANDA_BROKERS`, port 9092) — and neither +accepts a `--brokers` flag directly, both need `-X admin.hosts=...` / +`-X brokers=...`. Get this wrong and it doesn't error loudly: it just +retries the health check forever without ever reporting why. + ## In the full stack The root-level `docker-compose.yml` builds this directory's `Dockerfile` diff --git a/transport/provision-topics.sh b/transport/provision-topics.sh index ffcfafd..a7910c8 100755 --- a/transport/provision-topics.sh +++ b/transport/provision-topics.sh @@ -6,18 +6,28 @@ # sibling container on the root compose's network, or in CI. set -euo pipefail +# NOTE: confirmed by actually running this against a live Redpanda +# container -- neither `rpk cluster health` nor `rpk topic ...` accept a +# `--brokers` flag in this rpk version. Health checks hit the Admin API +# (-X admin.hosts=..., port 9644); topic commands hit the Kafka API +# (-X brokers=..., port 9092). Getting this wrong doesn't error loudly: +# `rpk cluster health --brokers ...` fails with "unknown flag" but that +# failure was swallowed by this script's own `> /dev/null 2>&1` retry +# loop, which just silently retried the malformed command forever instead +# of ever becoming healthy. BROKERS="${REDPANDA_BROKERS:-localhost:9092}" +ADMIN_HOSTS="${REDPANDA_ADMIN_HOSTS:-localhost:9644}" TOPIC="${REDPANDA_TOPIC:-sentry.logs.raw}" PARTITIONS="${REDPANDA_TOPIC_PARTITIONS:-6}" -echo "Waiting for Redpanda at ${BROKERS}..." -until rpk cluster health --brokers "${BROKERS}" --exit-when-healthy > /dev/null 2>&1; do +echo "Waiting for Redpanda admin API at ${ADMIN_HOSTS}..." +until rpk cluster health -X "admin.hosts=${ADMIN_HOSTS}" --exit-when-healthy > /dev/null 2>&1; do sleep 1 done -if rpk topic list --brokers "${BROKERS}" | awk 'NR>1{print $1}' | grep -qx "${TOPIC}"; then +if rpk topic list -X "brokers=${BROKERS}" | awk 'NR>1{print $1}' | grep -qx "${TOPIC}"; then echo "Topic '${TOPIC}' already exists, skipping." else echo "Creating topic '${TOPIC}' (${PARTITIONS} partitions)..." - rpk topic create "${TOPIC}" --brokers "${BROKERS}" --partitions "${PARTITIONS}" --replicas 1 + rpk topic create "${TOPIC}" -X "brokers=${BROKERS}" --partitions "${PARTITIONS}" --replicas 1 fi