From 5b7b119ceb68bbb0985cb33bacbee62fe07bbdb0 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Wed, 18 Nov 2026 22:09:22 -0700 Subject: [PATCH 1/2] --check: whether the data, blob store and coordinator can be used For the Helm chart's pre-flight init container (K-C5). It opens the data store as a start does, then writes, reads and deletes one small object in the blob store and connects to NATS once, without the background retrying the server itself does. Each part prints a line; one that can't be used says so as ': : ...' with K-S6's exit codes (78 bad configuration, 69 unreachable, 77 credentials refused, 76 TLS failed). It starts nothing and claims no node id. S3 failures are told apart by HTTP status, or the client's error when there was none; NATS by the connect error's kind. --- crates/common/src/manager/boot.rs | 23 ++- crates/common/src/manager/check.rs | 144 ++++++++++++++ crates/common/src/manager/mod.rs | 1 + crates/coordinator/src/backend/nats/mod.rs | 65 +++++++ crates/coordinator/src/bootstrap.rs | 16 ++ crates/store/src/registry/startup.rs | 213 +++++++++++++++++---- 6 files changed, 423 insertions(+), 39 deletions(-) create mode 100644 crates/common/src/manager/check.rs diff --git a/crates/common/src/manager/boot.rs b/crates/common/src/manager/boot.rs index 367e830e..02308570 100644 --- a/crates/common/src/manager/boot.rs +++ b/crates/common/src/manager/boot.rs @@ -67,6 +67,7 @@ Options: -u, --upgrade-store Upgrade the store's layout, then exit --seed Set up a new store from a file, then exit --upgrade-plan Print what --upgrade-store would do, then exit + --check Check the data, blob store and coordinator, then exit -h, --help Print help -V, --version Print version @@ -94,6 +95,14 @@ of JSON, the layout steps --upgrade-store would run, and exits: 0 when there are none, 3 when there are and every one can run with older nodes still up, 4 when one needs every other node stopped first, 2 when a newer release has changed the store, 1 when it can't tell (the message says why). + +--check opens the data store, then tests the blob store (writing, reading +and deleting one small object) and the coordinator (connecting to NATS), and +prints a line for each. It starts nothing. It exits 0 when all three can be +used; otherwise the failing part and why are on its last lines, and the exit +code says which kind of failure: 78 bad configuration, 69 unreachable, +77 credentials refused, 76 TLS failed, 1 anything else. A server start that +can't open the data store exits the same way. "# ); @@ -108,6 +117,8 @@ enum StoreOp { Seed(PathBuf), // inbuxa: K-C9 UpgradePlan, + // inbuxa: K-C5 + Check, None, } @@ -174,6 +185,9 @@ impl BootManager { ("upgrade-plan", None) => { import_export = StoreOp::UpgradePlan; } + ("check", None) => { + import_export = StoreOp::Check; + } ("seed", Some(value)) => { import_export = StoreOp::Seed(value.into()); } @@ -196,6 +210,11 @@ impl BootManager { failed("Missing '--config' argument (or CONFIG_PATH) for this command."); } + // inbuxa: K-C5: a check runs on its own and exits + if import_export == StoreOp::Check { + super::check::run(PathBuf::from(config_path.unwrap())).await; + } + // Initialize registry let upgrade_store = import_export == StoreOp::UpgradeStore; let seed = match &import_export { @@ -386,6 +405,8 @@ impl BootManager { core.queue_reindex(&imported).await; std::process::exit(0); } + // inbuxa: K-C5: ran and exited before the registry + StoreOp::Check => unreachable!(), StoreOp::Console => { // Store console store_console( @@ -432,7 +453,7 @@ pub fn build_ipc(has_pubsub: bool) -> (Ipc, IpcReceivers) { fn startup_failed(err: store::registry::startup::StartupError) -> ! { trc::event!( Server(trc::ServerEvent::StartupError), - Details = err.kind.label(), + Details = err.headline(), Reason = err.message.clone() ); diff --git a/crates/common/src/manager/check.rs b/crates/common/src/manager/check.rs new file mode 100644 index 00000000..ff91a652 --- /dev/null +++ b/crates/common/src/manager/check.rs @@ -0,0 +1,144 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +//! inbuxa: K-C5: `--check`, whether the data layer this server is set up for +//! can be used: the data store, the blob store and the coordinator, in that +//! order. It reads the settings and writes one small object to the blob +//! store, which it deletes again; it starts nothing and claims no node id. +//! +//! Each part prints a line. A part that fails says why, in the words and with +//! the exit code of `store::registry::startup`; with more than one failing, +//! the exit code is the first one's. The Helm chart runs it as an init +//! container, so a pod that can't use the data layer says which part, and +//! why, in its status. + +use coordinator::Coordinator; +use registry::schema::{enums::CompressionAlgo, structs}; +use std::path::PathBuf; +use store::{ + BlobStore, RegistryStore, + registry::{ + bootstrap::Bootstrap, + startup::{Part, StartupError, StartupFailure}, + }, +}; + +pub async fn run(config_path: PathBuf) -> ! { + let registry = match RegistryStore::init(config_path.clone(), false).await { + Ok(registry) => registry, + Err(err) => fail(err), + }; + if registry.is_bootstrap_mode() { + fail(StartupError::new( + StartupFailure::Config, + format!( + "there are no data store settings at {}", + config_path.display() + ), + )); + } + println!("ok data store"); + let mut bp = Bootstrap::new(registry).await; + + let mut failures = Vec::new(); + match blob_store(&mut bp).await { + Ok(found) => println!("ok blob store: {found}"), + Err(err) => failures.push(err), + } + match coordinator(&mut bp).await { + Ok(found) => println!("ok coordinator: {found}"), + Err(err) => failures.push(err), + } + + let code = failures.first().map(|err| err.kind.exit_code()); + for err in failures { + eprintln!("⚠️ Check failed: {err}"); + } + std::process::exit(code.unwrap_or(0)); +} + +fn fail(err: StartupError) -> ! { + eprintln!("⚠️ Check failed: {err}"); + std::process::exit(err.kind.exit_code()); +} + +/// The setting couldn't be read: a fault of the data store, which answered +/// a moment ago. +fn unreadable(part: Part, err: trc::Error) -> StartupError { + StartupError::of_part( + Part::DataStore, + StartupFailure::of(&err), + format!("reading the {} settings: {err}", part.name()), + ) +} + +async fn blob_store(bp: &mut Bootstrap) -> Result { + let part = Part::BlobStore; + let config = bp + .setting::() + .await + .map_err(|err| unreadable(part, err))?; + if matches!(config, structs::BlobStore::Default) { + return Ok("kept in the data store".into()); + } + let mut warnings = Vec::new(); + let store = BlobStore::open(config, &bp.data_store, &mut warnings) + .await + .map_err(|err| StartupError::of_part(part, StartupFailure::Config, err))?; + + // Everything the server does to a blob: write it, read it back, delete it + let key = format!( + ".inbuxa-check/{}-{}", + bp.registry.local_hostname(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or_default() + ); + let data = b"inbuxa --check"; + let failed = |what: &str, err: trc::Error| { + StartupError::of_part(part, StartupFailure::of(&err), format!("{what}: {err}")) + }; + store + .put_blob(key.as_bytes(), data, CompressionAlgo::None) + .await + .map_err(|err| failed("writing a test object", err))?; + let read = store + .get_blob(key.as_bytes(), 0..usize::MAX) + .await + .map_err(|err| failed("reading the test object back", err)); + let deleted = store + .delete_blob(key.as_bytes()) + .await + .map_err(|err| failed("deleting the test object", err)); + match read? { + Some(read) if read == data => {} + _ => { + return Err(StartupError::of_part( + part, + StartupFailure::Other, + format!("the test object {key} read back differently, or not at all"), + )); + } + } + deleted?; + Ok("wrote, read and deleted a test object".into()) +} + +async fn coordinator(bp: &mut Bootstrap) -> Result { + let part = Part::Coordinator; + let config = bp + .setting::() + .await + .map_err(|err| unreadable(part, err))?; + let none = matches!(config, structs::Coordinator::Disabled); + match Coordinator::probe(config).await { + Ok(Some(found)) => Ok(found), + Ok(None) if none => Ok("none, so this server can't run beside others".into()), + Ok(None) => Ok("not one this check can test".into()), + Err((kind, message)) => Err(StartupError::of_part(part, kind, message)), + } +} diff --git a/crates/common/src/manager/mod.rs b/crates/common/src/manager/mod.rs index 73d50e41..e7ad827a 100644 --- a/crates/common/src/manager/mod.rs +++ b/crates/common/src/manager/mod.rs @@ -19,6 +19,7 @@ pub mod application; pub mod backup; pub mod blob_moves; // inbuxa: blob-store-change spec, a blob store that moved pub mod boot; +pub mod check; // inbuxa: K-C5, --check pub mod compliance_roles; // inbuxa: personal-data catalog, the compliance roles pub mod console; pub mod defaults; diff --git a/crates/coordinator/src/backend/nats/mod.rs b/crates/coordinator/src/backend/nats/mod.rs index 8964e46e..0f6edbbe 100644 --- a/crates/coordinator/src/backend/nats/mod.rs +++ b/crates/coordinator/src/backend/nats/mod.rs @@ -17,6 +17,7 @@ use std::{ use crate::Coordinator; use async_nats::Client; use registry::schema::structs::NatsCoordinator; +use store::registry::startup::StartupFailure; use tokio::sync::watch; use trc::ClusterEvent; @@ -81,6 +82,70 @@ impl NatsPubSub { .map_err(|err| format!("Failed to connect to Nats: {}", err)) } + /// inbuxa: K-C5: connects once, with no retrying in the background as + /// `open` does, and says why it couldn't. For `--check`. + pub async fn probe(config: NatsCoordinator) -> Result { + if config.addresses.is_empty() { + return Err((StartupFailure::Config, "no NATS addresses are set".into())); + } + let addresses = config.addresses.into_inner(); + let named = addresses + .iter() + .map(|a| a.to_string()) + .collect::>() + .join(", "); + let timeout = config.timeout_connection.into_inner(); + let mut opts = async_nats::ConnectOptions::new() + .connection_timeout(timeout) + .max_reconnects(Some(1)) + .require_tls(config.use_tls); + let secret = |r: Result<_, String>| r.map_err(|err| (StartupFailure::Config, err)); + if let (Some(user), Some(pass)) = ( + config.auth_username, + secret(config.auth_secret.secret().await)?.map(|v| v.into_owned()), + ) { + opts = opts.user_and_password(user.to_string(), pass.to_string()); + } else if let Some(token) = + secret(config.credentials.secret().await)?.map(|v| v.into_owned()) + { + opts = opts.token(token); + } + + let connect = async { + let client = async_nats::connect_with_options(addresses, opts) + .await + .map_err(|err| { + use async_nats::ConnectErrorKind as K; + let kind = match err.kind() { + K::ServerParse => StartupFailure::Config, + K::Dns | K::Io | K::TimedOut | K::MaxReconnects => { + StartupFailure::Unreachable + } + K::Authentication | K::AuthorizationViolation => { + StartupFailure::Credentials + } + K::Tls => StartupFailure::Tls, + }; + (kind, format!("{named}: {err}")) + })?; + client + .flush() + .await + .map_err(|err| (StartupFailure::Unreachable, format!("{named}: {err}")))?; + let server = client.server_info(); + let _ = client.drain().await; + Ok(format!("NATS {} at {named}", server.version)) + }; + tokio::time::timeout(timeout * 2 + Duration::from_secs(5), connect) + .await + .unwrap_or_else(|_| { + Err(( + StartupFailure::Unreachable, + format!("{named}: no answer in {}s", (timeout * 2).as_secs() + 5), + )) + }) + } + /// inbuxa: counts the client's reconnections. Core NATS delivers at /// most once, so whatever the cluster published while this node was /// away is gone; a watcher drops what those events would have changed. diff --git a/crates/coordinator/src/bootstrap.rs b/crates/coordinator/src/bootstrap.rs index 8cd35d67..f05b7a8a 100644 --- a/crates/coordinator/src/bootstrap.rs +++ b/crates/coordinator/src/bootstrap.rs @@ -66,6 +66,22 @@ impl Coordinator { } } + /// inbuxa: K-C5: whether the coordinator a setting names answers, for + /// `--check`. `Ok(None)` when there's none, or it's of a kind not + /// checked here. + pub async fn probe( + config: structs::Coordinator, + ) -> Result, (store::registry::startup::StartupFailure, String)> { + match config { + structs::Coordinator::Disabled => Ok(None), + #[cfg(feature = "nats")] + structs::Coordinator::Nats(nats) => crate::backend::nats::NatsPubSub::probe(nats) + .await + .map(Some), + _ => Ok(None), + } + } + pub fn is_enabled(&self) -> bool { !matches!(self, Coordinator::None) } diff --git a/crates/store/src/registry/startup.rs b/crates/store/src/registry/startup.rs index ba978b89..9e0669e4 100644 --- a/crates/store/src/registry/startup.rs +++ b/crates/store/src/registry/startup.rs @@ -4,23 +4,24 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -//! inbuxa: K-S6: why the data store couldn't be opened at start-up. +//! inbuxa: K-S6, K-C5: why a part of the data layer couldn't be used. //! -//! Each case has its own exit code (from sysexits.h) and its own first words -//! on the error line, so a supervisor, a Kubernetes pre-flight check or a -//! person reading `kubectl describe` can tell them apart without parsing a -//! database driver's message: +//! Each case has its own exit code (from sysexits.h) and its own words on the +//! error line, after the part's name, so a supervisor, a Kubernetes pre-flight +//! check or a person reading `kubectl describe` can tell them apart without +//! parsing a driver's message: //! -//! | exit | line starts | meaning | -//! |------|--------------------------------------------|-------------------------------------------| -//! | 78 | `data store: bad configuration` | the settings file, a secret file it names, or the database name | -//! | 69 | `data store: unreachable` | no answer: refused, unknown host, timed out, or the server is starting or full | -//! | 77 | `data store: credentials refused` | wrong user or password, or the role lacks a permission it needs | -//! | 76 | `data store: TLS failed` | the TLS handshake failed: an unknown CA, or TLS the server doesn't offer | -//! | 1 | `data store: error` | anything else | +//! | exit | line starts | meaning | +//! |------|---------------------------------|-------------------------------------------| +//! | 78 | `: bad configuration` | the settings, a secret file they name, the database or bucket name | +//! | 69 | `: unreachable` | no answer: refused, unknown host, timed out, or the server is starting or full | +//! | 77 | `: credentials refused` | wrong user, password or key, or a permission it needs is missing | +//! | 76 | `: TLS failed` | the TLS handshake failed: an unknown CA, or TLS the server doesn't offer | +//! | 1 | `: error` | anything else | //! -//! PostgreSQL errors are told apart fully. Other stores report bad -//! configuration (78) and otherwise 1. +//! The part is `data store` (at every start, and `--check`), `blob store` or +//! `coordinator` (`--check` only). PostgreSQL, S3 and NATS errors are told +//! apart; other stores report bad configuration and otherwise 1. use std::fmt; @@ -33,6 +34,32 @@ pub enum StartupFailure { Other, } +/// Which part of the data layer failed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Part { + DataStore, + BlobStore, + Coordinator, +} + +impl Part { + pub fn name(self) -> &'static str { + match self { + Part::DataStore => "data store", + Part::BlobStore => "blob store", + Part::Coordinator => "coordinator", + } + } + + fn what(self) -> &'static str { + match self { + Part::DataStore => "the database", + Part::BlobStore => "the blob store", + Part::Coordinator => "NATS", + } + } +} + impl StartupFailure { pub fn exit_code(self) -> i32 { match self { @@ -44,41 +71,66 @@ impl StartupFailure { } } + /// The case, in the words that follow the part's name. pub fn label(self) -> &'static str { match self { - StartupFailure::Config => "data store: bad configuration", - StartupFailure::Unreachable => "data store: unreachable", - StartupFailure::Credentials => "data store: credentials refused", - StartupFailure::Tls => "data store: TLS failed", - StartupFailure::Other => "data store: error", + StartupFailure::Config => "bad configuration", + StartupFailure::Unreachable => "unreachable", + StartupFailure::Credentials => "credentials refused", + StartupFailure::Tls => "TLS failed", + StartupFailure::Other => "error", } } /// What to check, in a sentence. - pub fn hint(self) -> &'static str { - match self { - StartupFailure::Config => { + pub fn hint(self, part: Part) -> String { + let what = part.what(); + match (self, part) { + (StartupFailure::Config, Part::DataStore) => { "Check the data store settings file, the secret files it names and the database name." + .into() } - StartupFailure::Unreachable => { - "Check that the database is running and that its host and port are reachable from here." + (StartupFailure::Config, Part::BlobStore) => { + "Check the blob store settings: the bucket, its region and endpoint, and the secret files they name." + .into() } - StartupFailure::Credentials => { - "Check the user name and password, and that the user may use the database." + (StartupFailure::Config, Part::Coordinator) => { + "Check the coordinator settings and the secret files they name.".into() } - StartupFailure::Tls => { - "Check that the database offers TLS and that its certificate is trusted here." + (StartupFailure::Unreachable, _) => format!( + "Check that {what} is running and that its host and port are reachable from here." + ), + (StartupFailure::Credentials, Part::BlobStore) => { + "Check the access key and secret, and that they may write and delete in the bucket." + .into() } - StartupFailure::Other => "", + (StartupFailure::Credentials, Part::DataStore) => { + "Check the user name and password, and that the user may use the database.".into() + } + (StartupFailure::Credentials, Part::Coordinator) => { + "Check the user name and password, or the token.".into() + } + (StartupFailure::Tls, _) => { + format!("Check that {what} offers TLS and that its certificate is trusted here.") + } + (StartupFailure::Other, _) => String::new(), } } - /// What a data store error at start-up means. + /// What a data store error means. pub fn of(err: &trc::Error) -> Self { + let code = err.value_as_str(trc::Key::Code); + let reason = err.value_as_str(trc::Key::Reason).unwrap_or_default(); if err.matches(trc::EventType::Store(trc::StoreEvent::PostgresqlError)) { - Self::of_postgres( - err.value_as_str(trc::Key::Code), - err.value_as_str(trc::Key::Reason).unwrap_or_default(), + Self::of_postgres(code, reason) + } else if err.matches(trc::EventType::Store(trc::StoreEvent::S3Error)) { + Self::of_s3( + err.value(trc::Key::Code).and_then(|v| match v { + trc::Value::UInt(n) => Some(*n), + trc::Value::Int(n) => u64::try_from(*n).ok(), + _ => None, + }), + reason, ) } else { StartupFailure::Other @@ -106,27 +158,76 @@ impl StartupFailure { None => StartupFailure::Other, } } + + /// `status` is the HTTP status when the service answered; otherwise + /// `reason` is the HTTP client's chain of errors. + fn of_s3(status: Option, reason: &str) -> Self { + match status { + // A key the service doesn't know, a bad signature, or a policy + // that refuses the operation + Some(401 | 403) => StartupFailure::Credentials, + // No such bucket, or a bucket in another region or endpoint + Some(301 | 307 | 400 | 404) => StartupFailure::Config, + Some(429 | 500..=599) => StartupFailure::Unreachable, + Some(_) => StartupFailure::Other, + None => { + let reason = reason.to_ascii_lowercase(); + if ["certificate", "tls", "ssl", "handshake"] + .iter() + .any(|w| reason.contains(w)) + { + StartupFailure::Tls + } else if [ + "connect", + "dns error", + "failed to lookup", + "timed out", + "connection refused", + "connection reset", + ] + .iter() + .any(|w| reason.contains(w)) + { + StartupFailure::Unreachable + } else { + StartupFailure::Other + } + } + } + } } #[derive(Debug)] pub struct StartupError { + pub part: Part, pub kind: StartupFailure, pub message: String, } impl StartupError { + /// A data store failure. pub fn new(kind: StartupFailure, message: impl Into) -> Self { + Self::of_part(Part::DataStore, kind, message) + } + + pub fn of_part(part: Part, kind: StartupFailure, message: impl Into) -> Self { StartupError { + part, kind, message: message.into(), } } + + /// "data store: unreachable" + pub fn headline(&self) -> String { + format!("{}: {}", self.part.name(), self.kind.label()) + } } impl fmt::Display for StartupError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}: {}", self.kind.label(), self.message)?; - let hint = self.kind.hint(); + write!(f, "{}: {}", self.headline(), self.message)?; + let hint = self.kind.hint(self.part); if !hint.is_empty() { write!(f, "\n {hint}")?; } @@ -136,7 +237,10 @@ impl fmt::Display for StartupError { #[cfg(test)] mod tests { - use super::StartupFailure::{self, *}; + use super::{ + Part, + StartupFailure::{self, *}, + }; #[test] fn postgres_cases() { @@ -208,15 +312,48 @@ mod tests { } #[test] - fn codes_are_distinct() { + fn s3_cases() { + let cases: &[(Option, &str, StartupFailure)] = &[ + (Some(403), "InvalidAccessKeyId", Credentials), + (Some(403), "SignatureDoesNotMatch", Credentials), + (Some(404), "NoSuchBucket", Config), + (Some(301), "PermanentRedirect", Config), + (Some(503), "SlowDown", Unreachable), + ( + None, + "error sending request: client error (Connect): tcp connect error: Connection refused (os error 111)", + Unreachable, + ), + ( + None, + "error sending request: client error (Connect): dns error: failed to lookup address information", + Unreachable, + ), + ( + None, + "error sending request: client error (Connect): invalid peer certificate: UnknownIssuer", + Tls, + ), + (Some(418), "", Other), + ]; + for (status, reason, want) in cases { + assert_eq!(StartupFailure::of_s3(*status, reason), *want, "{reason}"); + } + } + + #[test] + fn codes_and_words_are_distinct() { let all = [Config, Unreachable, Credentials, Tls, Other]; for a in all { for b in all { if a != b { assert_ne!(a.exit_code(), b.exit_code()); - assert!(!a.label().starts_with(b.label()), "{}", a.label()); + assert_ne!(a.label(), b.label()); } } + for part in [Part::DataStore, Part::BlobStore, Part::Coordinator] { + assert_eq!(a.hint(part).is_empty(), a == Other); + } } } } -- 2.54.0 From ff145da31ebc402388e4a193c34cd4cf26959147 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Wed, 18 Nov 2026 23:14:10 -0700 Subject: [PATCH 2/2] Modification notice on bootstrap.rs --- crates/coordinator/src/bootstrap.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/coordinator/src/bootstrap.rs b/crates/coordinator/src/bootstrap.rs index f05b7a8a..4a0a42bb 100644 --- a/crates/coordinator/src/bootstrap.rs +++ b/crates/coordinator/src/bootstrap.rs @@ -2,6 +2,8 @@ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + * + * Modified by Coffey Labs in 2026 for INBUXA. */ use crate::Coordinator; -- 2.54.0