cairnobs/docker-compose.yml

# Phase 0+1 stack: Redpanda -> ingest -> ClickHouse -> api -> web, plus
# search (Tantivy full-text indexing, reads the same Redpanda topic
# ingest's consumer does).
#
# Does NOT include the Rust agent — see /agent/README.md: journald
# sourcing needs the host's journal, which isn't something a container
# gets for free. Run the agent natively on the host per
# /docs/phase-0-runbook.md, pointed at ingest's mapped port (localhost:4317).
# Windows Event Log/ETW sourcing needs a real Windows host regardless —
# see /docs/phase-1-runbook.md.
#
# Before first run: generate dev mTLS certs (hack/dev-certs/generate.sh).
# See /docs/phase-0-runbook.md (Linux pipeline) and
# /docs/phase-1-runbook.md (Windows + full-text search) for the full
# sequences.
services:
  redpanda:
    image: docker.redpanda.com/redpandadata/redpanda:v24.2.7
    container_name: cairnobs-redpanda
    command:
      - redpanda
      - start
      - --smp=1
      - --memory=1G
      - --reserve-memory=0M
      - --overprovisioned
      - --node-id=0
      - --check=false
      - --kafka-addr=PLAINTEXT://0.0.0.0:9092
      - --advertise-kafka-addr=PLAINTEXT://redpanda:9092
    ports:
      - "9092:9092"
    volumes:
      - redpanda-data:/var/lib/redpanda/data
    healthcheck:
      test: ["CMD", "rpk", "cluster", "health", "--exit-when-healthy"]
      interval: 5s
      timeout: 5s
      retries: 30

  # One-shot: creates the cairnobs.logs.raw topic, then exits 0. ingest
  # waits on this completing successfully before it starts.
  redpanda-provision:
    build:
      context: ./transport
    container_name: cairnobs-redpanda-provision
    depends_on:
      redpanda:
        condition: service_healthy
    environment:
      REDPANDA_BROKERS: "redpanda:9092"
      REDPANDA_ADMIN_HOSTS: "redpanda:9644"
      # Explicit rather than relying on both this script's and /search's
      # defaults happening to agree — search consumes this same topic and
      # needs to know the partition count up front (see /search/README.md).
      REDPANDA_TOPIC_PARTITIONS: "6"

  clickhouse:
    image: clickhouse/clickhouse-server:24.8
    container_name: cairnobs-clickhouse
    ports:
      - "8123:8123" # HTTP interface, used by the migrate step
      - "9000:9000" # native protocol, used by ingest and api
    environment:
      # The official image disables *network* access entirely for the
      # default user (even from sibling containers) unless
      # CLICKHOUSE_USER or CLICKHOUSE_PASSWORD is set to a genuinely
      # non-empty value — confirmed by testing, not just reading docs: an
      # explicitly-empty CLICKHOUSE_PASSWORD="" still triggers the
      # lockdown, silently returning 403 to every other container.
      #
      # TREAT THIS AS A REAL SECRET in any deployment. It began as
      # nothing more than a login gate — mTLS between agent and ingest
      # is still the security boundary that matters for *ingestion* —
      # but CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT below (added for Phase
      # 4's per-tenant provisioning) promoted the default user to a
      # ClickHouse superuser: `SHOW GRANTS FOR default` returns ACCESS
      # MANAGEMENT and CLUSTER ON *.* WITH GRANT OPTION. Whoever holds
      # this password can read and write every tenant's logs and mint
      # further users at will.
      #
      # The value below is a dev-only default and must be overridden via
      # docker-compose.override.yml (gitignored) anywhere real --
      # api/internal/config warns at startup if it is still in use.
      #
      # Rotating it is not just an env-var edit: the image's entrypoint
      # consumes this only at volume-init time, so on an existing volume
      # you must ALTER USER default IDENTIFIED BY '<new>' inside
      # ClickHouse first, then update the override and restart
      # clickhouse, api, and ingest -- the three services that carry it.
      CLICKHOUSE_PASSWORD: "cairnobs-dev-only"
      # Phase 4's per-tenant provisioning (enterprise/internal/tenantprovision)
      # runs CREATE USER/GRANT against this connection as the ClickHouse
      # admin -- the official image's default user doesn't have
      # access_management rights unless this is set, confirmed the hard
      # way: -provision-tenant failed with "Not enough privileges... grant
      # CREATE USER ON *.*" the first time this ran against a real
      # ClickHouse container, since every prior verification of
      # tenantprovision had been Docker-free (fakes) or never actually
      # exercised the admin connection this env var gates. The variable is
      # genuinely named CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT, not
      # CLICKHOUSE_ACCESS_MANAGEMENT -- confirmed by reading the image's
      # own /entrypoint.sh after the more obvious name silently did
      # nothing (no error, just left access_management="0" in the
      # generated users.d/default-user.xml).
      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1"
    volumes:
      - clickhouse-data:/var/lib/clickhouse
    ulimits:
      nofile:
        soft: 262144
        hard: 262144
    healthcheck:
      test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8123/ping"]
      interval: 5s
      timeout: 5s
      retries: 30

  # One-shot: applies /storage/migrations/*.sql, then exits 0. ingest and
  # api both wait on this completing successfully.
  clickhouse-migrate:
    build:
      context: ./storage
    container_name: cairnobs-clickhouse-migrate
    depends_on:
      clickhouse:
        condition: service_healthy
    environment:
      CLICKHOUSE_HTTP: "http://clickhouse:8123"
      CLICKHOUSE_PASSWORD: "cairnobs-dev-only"

  # Control-plane metadata store (dashboards, alert rules -- see
  # /docs/phase-3-dashboard-design.md for why this is Postgres rather
  # than new ClickHouse tables). Log data stays on ClickHouse/Tantivy
  # only, unaffected.
  metadata-postgres:
    image: postgres:16-alpine
    container_name: cairnobs-metadata-postgres
    environment:
      POSTGRES_DB: cairnobs_metadata
      POSTGRES_USER: cairnobs
      POSTGRES_PASSWORD: "cairnobs-dev-only" # not a real secret, same framing as CLICKHOUSE_PASSWORD above
    volumes:
      - metadata-postgres-data:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U cairnobs -d cairnobs_metadata"]
      interval: 5s
      timeout: 5s
      retries: 30

  # One-shot: applies /metadata/migrations/*.sql, then exits 0. api waits
  # on this completing successfully, same shape as clickhouse-migrate.
  metadata-migrate:
    build:
      context: ./metadata
    container_name: cairnobs-metadata-migrate
    depends_on:
      metadata-postgres:
        condition: service_healthy
    environment:
      POSTGRES_HOST: "metadata-postgres"
      POSTGRES_PORT: "5432"
      POSTGRES_USER: "cairnobs"
      POSTGRES_PASSWORD: "cairnobs-dev-only"
      POSTGRES_DATABASE: "cairnobs_metadata"
      # Password for the restricted audit_writer Postgres role (Phase 4
      # task 4) -- INSERT+SELECT only on audit_log, never UPDATE/DELETE,
      # via its own connection pool distinct from the shared "cairnobs"
      # role every other store uses. See /docs/phase-4-isolation-design.md.
      AUDIT_WRITER_PASSWORD: "audit-writer-dev-only"

  ingest:
    build:
      context: . # needs both ingest/ and proto/
      dockerfile: ingest/Dockerfile
    container_name: cairnobs-ingest
    depends_on:
      redpanda-provision:
        condition: service_completed_successfully
      clickhouse-migrate:
        condition: service_completed_successfully
      metadata-migrate:
        condition: service_completed_successfully
    ports:
      - "4317:4317" # gRPC, mTLS — this is what the host-run agent connects to
    environment:
      REDPANDA_BROKERS: "redpanda:9092"
      CLICKHOUSE_ADDR: "clickhouse:9000"
      CLICKHOUSE_PASSWORD: "cairnobs-dev-only"
      # TLS_*_FILE env vars are left at their defaults
      # (/etc/cairnobs-ingest/{server,server-key,ca}.pem) — matches where
      # the volume below mounts the generated dev certs.
      #
      # ENTERPRISE_AUTH_URL is deliberately NOT set here (see
      # ingest/internal/grpcserver's TenantResolver): with it unset,
      # PushBatch attaches no tenant_id header to any record, matching
      # every Phase 0-3 deployment's behavior. Setting it to
      # "http://enterprise-auth:8082" would require every agent to
      # present a valid `Authorization: Bearer <ingest token>` (minted
      # via `enterprise-auth -create-ingest-credential-tenant=<id>`) or
      # be refused outright -- not turned on here since nothing in this
      # compose file provisions one.
      #
      # AGENT_REGISTRY_POSTGRES_ADDR enables agent inventory/remote
      # config (see /docs/agent-management-design.md) -- same "cairnobs"
      # shared Postgres role api/dashboards already uses (agent
      # inventory carries no tamper-evidence requirement, unlike
      # audit_log's dedicated restricted role). Set here (unlike
      # ENTERPRISE_AUTH_URL above) since this feature has no multi-
      # tenancy prerequisite -- it works the same in single-tenant core.
      AGENT_REGISTRY_POSTGRES_ADDR: "metadata-postgres:5432"
      AGENT_REGISTRY_POSTGRES_USERNAME: "cairnobs"
      AGENT_REGISTRY_POSTGRES_PASSWORD: "cairnobs-dev-only"
    volumes:
      - ./hack/dev-certs/out:/etc/cairnobs-ingest:ro

  # Reads the same cairnobs.logs.raw topic ingest's consumer does (own
  # offset tracking, own failure domain — see /search/README.md) and
  # builds a Tantivy full-text index over the message field.
  search:
    build:
      context: . # needs both search/ and proto/
      dockerfile: search/Dockerfile
    container_name: cairnobs-search
    depends_on:
      redpanda-provision:
        condition: service_completed_successfully
    environment:
      REDPANDA_BROKERS: "redpanda:9092"
      REDPANDA_TOPIC_PARTITIONS: "6" # must match redpanda-provision's above
      # tracing-subscriber's default filter suppresses INFO without this
      # -- found by actually checking `docker compose logs search` and
      # seeing nothing, same silent-logging gap the agent had in Phase 0.
      RUST_LOG: "info"
      # ENTERPRISE_AUTH_URL/ENTERPRISE_AUTH_SERVICE_TOKEN are deliberately
      # NOT set here -- same reason enterprise-auth's own service below
      # doesn't wire alerting's API_SERVICE_TOKEN in by default: the
      # token can't be known ahead of time (mint it with
      # `enterprise-auth -mint-service-token search` after enterprise-auth
      # is up), so this is a manual step, not a default. Unset means
      # search/src/tenants.rs's ActiveTenantTracker never starts and
      # write-routing has no active-tenant gate, same as every deployment
      # before this tracker existed -- see search/README.md's "Per-tenant
      # indices" section for how to turn it on for manual testing.
    volumes:
      - search-index-data:/var/lib/cairnobs-search

  # Mutually exclusive with enterprise-api below, same choice Helm makes
  # via enterprise.enabled (deploy/helm/cairnobs/templates/api.yaml vs
  # enterprise-api.yaml) -- selected by the COMPOSE_PROFILES value in
  # .env (checked in as "single-tenant", the zero-config default) or an
  # override on the command line, e.g. `COMPOSE_PROFILES=enterprise
  # docker compose up`. `docker compose run api ...` (as the manual RBAC
  # testing steps in enterprise/README.md/phase-4-runbook.md §4 use)
  # still works regardless of the active profile -- an explicit service
  # reference on the command line bypasses profile filtering.
  api:
    profiles: ["single-tenant"]
    build:
      context: . # needs both api/ and proto/ (gRPC client to search)
      dockerfile: api/Dockerfile
    container_name: cairnobs-api
    depends_on:
      clickhouse-migrate:
        condition: service_completed_successfully
      metadata-migrate:
        condition: service_completed_successfully
    ports:
      - "8080:8080"
    environment:
      CLICKHOUSE_ADDR: "clickhouse:9000"
      CLICKHOUSE_PASSWORD: "cairnobs-dev-only"
      SEARCH_GRPC_ADDR: "search:50052"
      POSTGRES_ADDR: "metadata-postgres:5432"
      POSTGRES_DATABASE: "cairnobs_metadata"
      POSTGRES_USERNAME: "cairnobs"
      POSTGRES_PASSWORD: "cairnobs-dev-only"
    healthcheck:
      # alerting (Phase 3 task 5) depends_on api -- without this, that
      # dependency can only mean "container started," not "actually
      # listening," and would hammer a not-yet-ready api with errors on
      # every evaluator tick during stack startup. api's image is
      # distroless (no shell, no wget) so this execs the api binary's own
      # -healthcheck self-check mode instead of an external tool.
      test: ["CMD", "/api", "-healthcheck"]
      interval: 5s
      timeout: 5s
      retries: 30

  alerting:
    build:
      context: alerting # self-contained, no /proto needed -- see alerting/Dockerfile
      dockerfile: Dockerfile
    container_name: cairnobs-alerting
    depends_on:
      metadata-migrate:
        condition: service_completed_successfully
      # Both optional (required: false): whichever of api/enterprise-api
      # is actually in the active profile set is the one this waits on
      # -- the other isn't defined for this run at all, and without
      # `required: false` compose would error on the inactive one rather
      # than just skipping it. See api's doc comment above.
      api:
        condition: service_healthy
        required: false
      enterprise-api:
        condition: service_healthy
        required: false
    ports:
      - "8081:8081"
    environment:
      POSTGRES_ADDR: "metadata-postgres:5432"
      POSTGRES_DATABASE: "cairnobs_metadata"
      POSTGRES_USERNAME: "cairnobs"
      POSTGRES_PASSWORD: "cairnobs-dev-only"
      # Resolves to whichever of api/enterprise-api is actually active --
      # enterprise-api declares a `default.aliases: [api]` network alias
      # below specifically so this never needs to change based on which
      # profile is selected.
      API_QUERY_URL: "http://api:8080"
    healthcheck:
      test: ["CMD", "/alerting", "-healthcheck"]
      interval: 5s
      timeout: 5s
      retries: 30

  # Commercial-license SSO/RBAC service (Phase 4) -- see
  # /docs/phase-4-isolation-design.md and enterprise/README.md. Included
  # here so it can be built/run/curled like every other service, but
  # deliberately NOT wired into api's ENTERPRISE_AUTH_URL or alerting's
  # API_SERVICE_TOKEN below: turning that on makes every /query and
  # /dashboards request require a valid session/service token. Both
  # OIDC and SAML login flows now exist (enterprise/internal/loginhandler),
  # but this compose file sets neither OIDC_ISSUER_URL nor
  # SAML_IDP_METADATA_URL, so both stay disabled here, and there's still
  # no admin UI to create the first tenant_memberships row -- see
  # /docs/phase-4-runbook.md sections 3a/3b for wiring a real IdP and
  # bootstrapping that row by hand. Flipping enforcement on by default
  # without that would break the web UI and cairnobsctl with no way to log
  # in. See enterprise/README.md for how to turn enforcement on for
  # manual testing (mint a service token, set the two env vars, restart).
  enterprise-auth:
    build:
      context: . # needs api/, ingest/, proto/, and enterprise/ itself -- see enterprise/Dockerfile's doc comment
      dockerfile: enterprise/Dockerfile
    container_name: cairnobs-enterprise-auth
    depends_on:
      metadata-migrate:
        condition: service_completed_successfully
    ports:
      - "8082:8082"
    environment:
      # Dev-only, same framing as CLICKHOUSE_PASSWORD above -- not a real
      # secret. Must be at least 32 bytes (see internal/config.Load).
      ENTERPRISE_SESSION_SIGNING_KEY: "cairnobs-dev-only-session-signing-key-32bytes+"
      POSTGRES_ADDR: "metadata-postgres:5432"
      POSTGRES_DATABASE: "cairnobs_metadata"
      POSTGRES_USERNAME: "cairnobs"
      POSTGRES_PASSWORD: "cairnobs-dev-only"
      # Where the browser lands after internal/loginhandler sets a
      # session cookie -- web's mapped host port (see web's build args
      # for why this is localhost:3000, not the compose network's
      # service DNS name: the browser resolves this, not a sibling
      # container).
      POST_LOGIN_REDIRECT_URL: "http://localhost:3000"
    healthcheck:
      test: ["CMD", "/enterprise-auth", "-healthcheck"]
      interval: 5s
      timeout: 5s
      retries: 30

  # Multi-tenant-aware alternative to `api` (Phase 4) -- see
  # enterprise/cmd/enterprise-api/main.go's doc comment for why this is
  # a second binary rather than a flag on `api`. Mutually exclusive with
  # `api` above via COMPOSE_PROFILES (see that service's doc comment);
  # when the "enterprise" profile is active this replaces `api` in the
  # traffic path transparently, same as Helm: HTTP_LISTEN_ADDR is
  # overridden to :8080 (this binary's own default is :8083) and the
  # `default.aliases` entry below makes this reachable at the hostname
  # `api` too, so alerting's API_QUERY_URL and web's VITE_API_BASE_URL
  # need zero conditional logic -- whichever binary is actually running
  # transparently answers on the same name/port either way. Nothing here
  # provisions any tenants on its own (see -provision-tenant).
  # CLICKHOUSE_ADMIN_USERNAME/PASSWORD reuse the same admin credential
  # `clickhouse-migrate` uses, since tenantprovision needs
  # access_management, not a tenant-scoped grant.
  enterprise-api:
    profiles: ["enterprise"]
    build:
      context: .
      dockerfile: enterprise/cmd/enterprise-api/Dockerfile
    container_name: cairnobs-enterprise-api
    depends_on:
      clickhouse-migrate:
        condition: service_completed_successfully
      metadata-migrate:
        condition: service_completed_successfully
    networks:
      default:
        aliases:
          - api
    ports:
      - "8080:8080"
    environment:
      HTTP_LISTEN_ADDR: ":8080"
      CLICKHOUSE_ADDR: "clickhouse:9000"
      CLICKHOUSE_ADMIN_USERNAME: "default"
      CLICKHOUSE_ADMIN_PASSWORD: "cairnobs-dev-only"
      SEARCH_GRPC_ADDR: "search:50052"
      POSTGRES_ADDR: "metadata-postgres:5432"
      POSTGRES_DATABASE: "cairnobs_metadata"
      POSTGRES_USERNAME: "cairnobs"
      POSTGRES_PASSWORD: "cairnobs-dev-only"
      AUDIT_WRITER_USERNAME: "audit_writer"
      AUDIT_WRITER_PASSWORD: "audit-writer-dev-only"
      ENTERPRISE_AUTH_URL: "http://enterprise-auth:8082"
    healthcheck:
      test: ["CMD", "/enterprise-api", "-healthcheck"]
      interval: 5s
      timeout: 5s
      retries: 30

  # Per-tenant write-routing for ingest (enterprise/internal/chwriter) --
  # see enterprise/cmd/enterprise-ingest/main.go's doc comment. Unlike
  # api/enterprise-api's COMPOSE_PROFILES trick, this is NOT wired to be
  # mutually exclusive with `ingest`'s own consumer half in this file:
  # `ingest` always runs -mode=all here regardless of profile (splitting
  # its server/consumer halves into separate compose services isn't done
  # -- real, disclosed scope, see /docs/phase-4-runbook.md), so with the
  # "enterprise" profile active both this service AND ingest's own
  # consumer independently read every message (different consumer
  # groups) and write it -- ingest into the one shared `logs` table,
  # this into each tenant's own database. Harmless duplication for local
  # testing/verification purposes, not what a real deployment does (see
  # deploy/helm/cairnobs's ingest.yaml/enterprise-ingest.yaml, which
  # actually achieve exclusivity via -mode=server/-mode=consumer).
  enterprise-ingest:
    profiles: ["enterprise"]
    build:
      context: .
      dockerfile: enterprise/cmd/enterprise-ingest/Dockerfile
    container_name: cairnobs-enterprise-ingest
    depends_on:
      redpanda-provision:
        condition: service_completed_successfully
      clickhouse-migrate:
        condition: service_completed_successfully
      metadata-migrate:
        condition: service_completed_successfully
    environment:
      REDPANDA_BROKERS: "redpanda:9092"
      CLICKHOUSE_ADDR: "clickhouse:9000"
      POSTGRES_ADDR: "metadata-postgres:5432"
      POSTGRES_DATABASE: "cairnobs_metadata"
      POSTGRES_USERNAME: "cairnobs"
      POSTGRES_PASSWORD: "cairnobs-dev-only"
    healthcheck:
      test: ["CMD", "/enterprise-ingest", "-healthcheck"]
      interval: 5s
      timeout: 5s
      retries: 30

  web:
    build:
      context: web
      args:
        # Baked in at build time (static site, not a server) as
        # localhost:8080/8081 -- fetched from the *browser*, which
        # resolves against the host's mapped ports, not the compose
        # network's service DNS names.
        # localhost:8080 works unchanged regardless of which profile is
        # active -- enterprise-api maps the same host port api does when
        # it's the one running (see that service's doc comment).
        VITE_API_BASE_URL: "http://localhost:8080"
        VITE_ALERTING_API_BASE_URL: "http://localhost:8081"
        VITE_ENTERPRISE_AUTH_BASE_URL: "http://localhost:8082"
    container_name: cairnobs-web
    depends_on:
      # api/enterprise-api optional, same reasoning as alerting's
      # depends_on above -- only one is ever in the active profile set.
      api:
        condition: service_started
        required: false
      enterprise-api:
        condition: service_started
        required: false
      alerting:
        condition: service_started
      enterprise-auth:
        condition: service_started
    ports:
      - "3000:3000"

volumes:
  redpanda-data:
  clickhouse-data:
  search-index-data:
  metadata-postgres-data: