jcoffey-dev
088677643f
search/src/consumer.rs's write-routing (built last pass) had no active- tenant check at all: IndexRegistry.resolve() would open-or-create an index directory for any syntactically-valid tenant_id, active or not -- unlike ClickHouse's chwriter.Registry (an active-tenants-only snapshot built at enterprise-ingest startup) or the read side (gated by searchclient.TenantChecker, a direct rbacstore query). search is AGPL core with no Postgres access and no enterprise/ import allowed, so it needed a network boundary instead -- the same shape ingest's TenantResolver already uses against enterprise-auth, just Rust calling Go instead of Go calling Go. New GET /internal/active-tenants endpoint on enterprise-auth (rbacstore.ListActiveTenantIDs + authhandler.handleActiveTenants), gated on a RoleService Bearer credential -- server-to-server auth, the same shape alerting presents to api, minted via the already-generic enterprise-auth -mint-service-token search. search/src/tenants.rs's ActiveTenantTracker polls it every 60s, blocking startup on the first fetch succeeding (fail-closed cold start -- a control-plane outage at boot must not silently accept every tenant_id) and keeping the last- known-good set on any later refresh failure (a transient blip shouldn't stop every tenant's indexing, only prevent the allowlist from growing/ shrinking until connectivity resumes). consumer.rs refuses any tagged record whose tenant isn't in the polled set, before ever calling resolve() -- IndexRegistry itself stays policy-free, matching the same mechanism/policy split clickhousewriter.Writer vs. chwriter.Registry already draws on the ClickHouse side. Off unless ENTERPRISE_AUTH_URL/ENTERPRISE_AUTH_SERVICE_TOKEN are both set (search/src/config.rs rejects exactly one being set) -- every existing deployment is unaffected. Verified with real HTTP round trips in this environment: tenants.rs's tests exercise real reqwest requests (actual Authorization: Bearer header, actual JSON parsing) against a hand-rolled dependency-free TCP test server, including both fail-closed paths (rejected first fetch, unreachable server). authhandler's new tests cover the credential-kind distinction this endpoint exists to enforce -- a real human session, even for a genuine Owner, must not satisfy a check meant for a service identity. One asymmetry remains, disclosed rather than fixed: chwriter.Registry's snapshot still never refreshes (stale until enterprise-ingest restarts), while ActiveTenantTracker's 60s poll gives Tantivy a materially tighter staleness window. Neither is a live per-write check -- that would mean a database/HTTP round trip per record, a throughput cost neither implementation accepts -- so both have some staleness window by design; the gap between the two windows is what's disclosed, not a claim either is fully live.
95 lines
3.3 KiB
Rust
95 lines
3.3 KiB
Rust
mod config;
|
|
mod consumer;
|
|
mod grpc;
|
|
mod index;
|
|
mod offsets;
|
|
mod registry;
|
|
mod tenants;
|
|
|
|
pub mod logsv1 {
|
|
tonic::include_proto!("sentry.logs.v1");
|
|
}
|
|
pub mod searchv1 {
|
|
tonic::include_proto!("sentry.search.v1");
|
|
}
|
|
|
|
use anyhow::{Context, Result};
|
|
use config::Config;
|
|
use index::SearchIndex;
|
|
use registry::IndexRegistry;
|
|
use std::sync::Arc;
|
|
use tonic::transport::Server;
|
|
|
|
/// Matches /transport/provision-topics.sh's default
|
|
/// REDPANDA_TOPIC_PARTITIONS -- documented cross-component contract, not
|
|
/// discovered dynamically. See consumer.rs.
|
|
const DEFAULT_PARTITION_COUNT: i32 = 6;
|
|
|
|
#[tokio::main]
|
|
async fn main() -> Result<()> {
|
|
tracing_subscriber::fmt()
|
|
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
|
|
.init();
|
|
|
|
let cfg = Arc::new(Config::load().context("loading config")?);
|
|
|
|
// Off unless ENTERPRISE_AUTH_URL/ENTERPRISE_AUTH_SERVICE_TOKEN are
|
|
// both set (Config::load already rejects exactly one being set).
|
|
// Blocks startup entirely on failure, same "fail hard, let the
|
|
// orchestrator restart" posture enterprise-ingest's main.go already
|
|
// uses when its own required startup fetch (rbacstore.
|
|
// ListProvisionedDataSources) fails -- see tenants.rs's doc comment
|
|
// for why a partial/degraded startup isn't the safer choice here.
|
|
let active_tenants = match (&cfg.enterprise_auth_url, &cfg.enterprise_auth_service_token) {
|
|
(Some(url), Some(token)) => {
|
|
tracing::info!(url, "active-tenant write-routing gate enabled");
|
|
Some(tenants::ActiveTenantTracker::start(url, token).await?)
|
|
}
|
|
_ => {
|
|
tracing::info!("ENTERPRISE_AUTH_URL not set -- write-routing has no active-tenant gate");
|
|
None
|
|
}
|
|
};
|
|
|
|
let index = Arc::new(
|
|
SearchIndex::open_or_create(&cfg.index_path).context("opening tantivy index")?,
|
|
);
|
|
// Per-tenant indices are resolved on demand by IndexRegistry, opened
|
|
// under cfg.tenants_index_path -- see registry.rs's doc comment for
|
|
// what this isolates (both read and write now) and the one residual
|
|
// gap it doesn't close.
|
|
let registry = Arc::new(IndexRegistry::new(Arc::clone(&index), cfg.tenants_index_path.clone()));
|
|
|
|
let partition_count: i32 = std::env::var("REDPANDA_TOPIC_PARTITIONS")
|
|
.ok()
|
|
.and_then(|s| s.parse().ok())
|
|
.unwrap_or(DEFAULT_PARTITION_COUNT);
|
|
|
|
let consumer_cfg = Arc::clone(&cfg);
|
|
let consumer_registry = Arc::clone(®istry);
|
|
let consumer_active_tenants = active_tenants.clone();
|
|
let consumer_handle = tokio::spawn(async move {
|
|
if let Err(e) = consumer::run(consumer_cfg, consumer_registry, partition_count, consumer_active_tenants).await {
|
|
tracing::error!(error = %e, "redpanda consumer exited with error");
|
|
}
|
|
});
|
|
|
|
let addr = cfg
|
|
.grpc_listen_addr
|
|
.parse()
|
|
.context("parsing GRPC_LISTEN_ADDR")?;
|
|
tracing::info!(addr = %cfg.grpc_listen_addr, "search gRPC server listening");
|
|
|
|
let search_server = grpc::SearchServer::new(Arc::clone(®istry));
|
|
Server::builder()
|
|
.add_service(searchv1::search_service_server::SearchServiceServer::new(
|
|
search_server,
|
|
))
|
|
.serve(addr)
|
|
.await
|
|
.context("gRPC server failed")?;
|
|
|
|
consumer_handle.abort();
|
|
Ok(())
|
|
}
|