diff --git a/crates/common/src/manager/boot.rs b/crates/common/src/manager/boot.rs index 367e830e..02308570 100644 --- a/crates/common/src/manager/boot.rs +++ b/crates/common/src/manager/boot.rs @@ -67,6 +67,7 @@ Options: -u, --upgrade-store Upgrade the store's layout, then exit --seed Set up a new store from a file, then exit --upgrade-plan Print what --upgrade-store would do, then exit + --check Check the data, blob store and coordinator, then exit -h, --help Print help -V, --version Print version @@ -94,6 +95,14 @@ of JSON, the layout steps --upgrade-store would run, and exits: 0 when there are none, 3 when there are and every one can run with older nodes still up, 4 when one needs every other node stopped first, 2 when a newer release has changed the store, 1 when it can't tell (the message says why). + +--check opens the data store, then tests the blob store (writing, reading +and deleting one small object) and the coordinator (connecting to NATS), and +prints a line for each. It starts nothing. It exits 0 when all three can be +used; otherwise the failing part and why are on its last lines, and the exit +code says which kind of failure: 78 bad configuration, 69 unreachable, +77 credentials refused, 76 TLS failed, 1 anything else. A server start that +can't open the data store exits the same way. "# ); @@ -108,6 +117,8 @@ enum StoreOp { Seed(PathBuf), // inbuxa: K-C9 UpgradePlan, + // inbuxa: K-C5 + Check, None, } @@ -174,6 +185,9 @@ impl BootManager { ("upgrade-plan", None) => { import_export = StoreOp::UpgradePlan; } + ("check", None) => { + import_export = StoreOp::Check; + } ("seed", Some(value)) => { import_export = StoreOp::Seed(value.into()); } @@ -196,6 +210,11 @@ impl BootManager { failed("Missing '--config' argument (or CONFIG_PATH) for this command."); } + // inbuxa: K-C5: a check runs on its own and exits + if import_export == StoreOp::Check { + super::check::run(PathBuf::from(config_path.unwrap())).await; + } + // Initialize registry let upgrade_store = import_export == StoreOp::UpgradeStore; let seed = match &import_export { @@ -386,6 +405,8 @@ impl BootManager { core.queue_reindex(&imported).await; std::process::exit(0); } + // inbuxa: K-C5: ran and exited before the registry + StoreOp::Check => unreachable!(), StoreOp::Console => { // Store console store_console( @@ -432,7 +453,7 @@ pub fn build_ipc(has_pubsub: bool) -> (Ipc, IpcReceivers) { fn startup_failed(err: store::registry::startup::StartupError) -> ! { trc::event!( Server(trc::ServerEvent::StartupError), - Details = err.kind.label(), + Details = err.headline(), Reason = err.message.clone() ); diff --git a/crates/common/src/manager/check.rs b/crates/common/src/manager/check.rs new file mode 100644 index 00000000..ff91a652 --- /dev/null +++ b/crates/common/src/manager/check.rs @@ -0,0 +1,144 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +//! inbuxa: K-C5: `--check`, whether the data layer this server is set up for +//! can be used: the data store, the blob store and the coordinator, in that +//! order. It reads the settings and writes one small object to the blob +//! store, which it deletes again; it starts nothing and claims no node id. +//! +//! Each part prints a line. A part that fails says why, in the words and with +//! the exit code of `store::registry::startup`; with more than one failing, +//! the exit code is the first one's. The Helm chart runs it as an init +//! container, so a pod that can't use the data layer says which part, and +//! why, in its status. + +use coordinator::Coordinator; +use registry::schema::{enums::CompressionAlgo, structs}; +use std::path::PathBuf; +use store::{ + BlobStore, RegistryStore, + registry::{ + bootstrap::Bootstrap, + startup::{Part, StartupError, StartupFailure}, + }, +}; + +pub async fn run(config_path: PathBuf) -> ! { + let registry = match RegistryStore::init(config_path.clone(), false).await { + Ok(registry) => registry, + Err(err) => fail(err), + }; + if registry.is_bootstrap_mode() { + fail(StartupError::new( + StartupFailure::Config, + format!( + "there are no data store settings at {}", + config_path.display() + ), + )); + } + println!("ok data store"); + let mut bp = Bootstrap::new(registry).await; + + let mut failures = Vec::new(); + match blob_store(&mut bp).await { + Ok(found) => println!("ok blob store: {found}"), + Err(err) => failures.push(err), + } + match coordinator(&mut bp).await { + Ok(found) => println!("ok coordinator: {found}"), + Err(err) => failures.push(err), + } + + let code = failures.first().map(|err| err.kind.exit_code()); + for err in failures { + eprintln!("⚠️ Check failed: {err}"); + } + std::process::exit(code.unwrap_or(0)); +} + +fn fail(err: StartupError) -> ! { + eprintln!("⚠️ Check failed: {err}"); + std::process::exit(err.kind.exit_code()); +} + +/// The setting couldn't be read: a fault of the data store, which answered +/// a moment ago. +fn unreadable(part: Part, err: trc::Error) -> StartupError { + StartupError::of_part( + Part::DataStore, + StartupFailure::of(&err), + format!("reading the {} settings: {err}", part.name()), + ) +} + +async fn blob_store(bp: &mut Bootstrap) -> Result { + let part = Part::BlobStore; + let config = bp + .setting::() + .await + .map_err(|err| unreadable(part, err))?; + if matches!(config, structs::BlobStore::Default) { + return Ok("kept in the data store".into()); + } + let mut warnings = Vec::new(); + let store = BlobStore::open(config, &bp.data_store, &mut warnings) + .await + .map_err(|err| StartupError::of_part(part, StartupFailure::Config, err))?; + + // Everything the server does to a blob: write it, read it back, delete it + let key = format!( + ".inbuxa-check/{}-{}", + bp.registry.local_hostname(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or_default() + ); + let data = b"inbuxa --check"; + let failed = |what: &str, err: trc::Error| { + StartupError::of_part(part, StartupFailure::of(&err), format!("{what}: {err}")) + }; + store + .put_blob(key.as_bytes(), data, CompressionAlgo::None) + .await + .map_err(|err| failed("writing a test object", err))?; + let read = store + .get_blob(key.as_bytes(), 0..usize::MAX) + .await + .map_err(|err| failed("reading the test object back", err)); + let deleted = store + .delete_blob(key.as_bytes()) + .await + .map_err(|err| failed("deleting the test object", err)); + match read? { + Some(read) if read == data => {} + _ => { + return Err(StartupError::of_part( + part, + StartupFailure::Other, + format!("the test object {key} read back differently, or not at all"), + )); + } + } + deleted?; + Ok("wrote, read and deleted a test object".into()) +} + +async fn coordinator(bp: &mut Bootstrap) -> Result { + let part = Part::Coordinator; + let config = bp + .setting::() + .await + .map_err(|err| unreadable(part, err))?; + let none = matches!(config, structs::Coordinator::Disabled); + match Coordinator::probe(config).await { + Ok(Some(found)) => Ok(found), + Ok(None) if none => Ok("none, so this server can't run beside others".into()), + Ok(None) => Ok("not one this check can test".into()), + Err((kind, message)) => Err(StartupError::of_part(part, kind, message)), + } +} diff --git a/crates/common/src/manager/mod.rs b/crates/common/src/manager/mod.rs index 73d50e41..e7ad827a 100644 --- a/crates/common/src/manager/mod.rs +++ b/crates/common/src/manager/mod.rs @@ -19,6 +19,7 @@ pub mod application; pub mod backup; pub mod blob_moves; // inbuxa: blob-store-change spec, a blob store that moved pub mod boot; +pub mod check; // inbuxa: K-C5, --check pub mod compliance_roles; // inbuxa: personal-data catalog, the compliance roles pub mod console; pub mod defaults; diff --git a/crates/coordinator/src/backend/nats/mod.rs b/crates/coordinator/src/backend/nats/mod.rs index 8964e46e..0f6edbbe 100644 --- a/crates/coordinator/src/backend/nats/mod.rs +++ b/crates/coordinator/src/backend/nats/mod.rs @@ -17,6 +17,7 @@ use std::{ use crate::Coordinator; use async_nats::Client; use registry::schema::structs::NatsCoordinator; +use store::registry::startup::StartupFailure; use tokio::sync::watch; use trc::ClusterEvent; @@ -81,6 +82,70 @@ impl NatsPubSub { .map_err(|err| format!("Failed to connect to Nats: {}", err)) } + /// inbuxa: K-C5: connects once, with no retrying in the background as + /// `open` does, and says why it couldn't. For `--check`. + pub async fn probe(config: NatsCoordinator) -> Result { + if config.addresses.is_empty() { + return Err((StartupFailure::Config, "no NATS addresses are set".into())); + } + let addresses = config.addresses.into_inner(); + let named = addresses + .iter() + .map(|a| a.to_string()) + .collect::>() + .join(", "); + let timeout = config.timeout_connection.into_inner(); + let mut opts = async_nats::ConnectOptions::new() + .connection_timeout(timeout) + .max_reconnects(Some(1)) + .require_tls(config.use_tls); + let secret = |r: Result<_, String>| r.map_err(|err| (StartupFailure::Config, err)); + if let (Some(user), Some(pass)) = ( + config.auth_username, + secret(config.auth_secret.secret().await)?.map(|v| v.into_owned()), + ) { + opts = opts.user_and_password(user.to_string(), pass.to_string()); + } else if let Some(token) = + secret(config.credentials.secret().await)?.map(|v| v.into_owned()) + { + opts = opts.token(token); + } + + let connect = async { + let client = async_nats::connect_with_options(addresses, opts) + .await + .map_err(|err| { + use async_nats::ConnectErrorKind as K; + let kind = match err.kind() { + K::ServerParse => StartupFailure::Config, + K::Dns | K::Io | K::TimedOut | K::MaxReconnects => { + StartupFailure::Unreachable + } + K::Authentication | K::AuthorizationViolation => { + StartupFailure::Credentials + } + K::Tls => StartupFailure::Tls, + }; + (kind, format!("{named}: {err}")) + })?; + client + .flush() + .await + .map_err(|err| (StartupFailure::Unreachable, format!("{named}: {err}")))?; + let server = client.server_info(); + let _ = client.drain().await; + Ok(format!("NATS {} at {named}", server.version)) + }; + tokio::time::timeout(timeout * 2 + Duration::from_secs(5), connect) + .await + .unwrap_or_else(|_| { + Err(( + StartupFailure::Unreachable, + format!("{named}: no answer in {}s", (timeout * 2).as_secs() + 5), + )) + }) + } + /// inbuxa: counts the client's reconnections. Core NATS delivers at /// most once, so whatever the cluster published while this node was /// away is gone; a watcher drops what those events would have changed. diff --git a/crates/coordinator/src/bootstrap.rs b/crates/coordinator/src/bootstrap.rs index 8cd35d67..4a0a42bb 100644 --- a/crates/coordinator/src/bootstrap.rs +++ b/crates/coordinator/src/bootstrap.rs @@ -2,6 +2,8 @@ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + * + * Modified by Coffey Labs in 2026 for INBUXA. */ use crate::Coordinator; @@ -66,6 +68,22 @@ impl Coordinator { } } + /// inbuxa: K-C5: whether the coordinator a setting names answers, for + /// `--check`. `Ok(None)` when there's none, or it's of a kind not + /// checked here. + pub async fn probe( + config: structs::Coordinator, + ) -> Result, (store::registry::startup::StartupFailure, String)> { + match config { + structs::Coordinator::Disabled => Ok(None), + #[cfg(feature = "nats")] + structs::Coordinator::Nats(nats) => crate::backend::nats::NatsPubSub::probe(nats) + .await + .map(Some), + _ => Ok(None), + } + } + pub fn is_enabled(&self) -> bool { !matches!(self, Coordinator::None) } diff --git a/crates/store/src/registry/startup.rs b/crates/store/src/registry/startup.rs index ba978b89..9e0669e4 100644 --- a/crates/store/src/registry/startup.rs +++ b/crates/store/src/registry/startup.rs @@ -4,23 +4,24 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -//! inbuxa: K-S6: why the data store couldn't be opened at start-up. +//! inbuxa: K-S6, K-C5: why a part of the data layer couldn't be used. //! -//! Each case has its own exit code (from sysexits.h) and its own first words -//! on the error line, so a supervisor, a Kubernetes pre-flight check or a -//! person reading `kubectl describe` can tell them apart without parsing a -//! database driver's message: +//! Each case has its own exit code (from sysexits.h) and its own words on the +//! error line, after the part's name, so a supervisor, a Kubernetes pre-flight +//! check or a person reading `kubectl describe` can tell them apart without +//! parsing a driver's message: //! -//! | exit | line starts | meaning | -//! |------|--------------------------------------------|-------------------------------------------| -//! | 78 | `data store: bad configuration` | the settings file, a secret file it names, or the database name | -//! | 69 | `data store: unreachable` | no answer: refused, unknown host, timed out, or the server is starting or full | -//! | 77 | `data store: credentials refused` | wrong user or password, or the role lacks a permission it needs | -//! | 76 | `data store: TLS failed` | the TLS handshake failed: an unknown CA, or TLS the server doesn't offer | -//! | 1 | `data store: error` | anything else | +//! | exit | line starts | meaning | +//! |------|---------------------------------|-------------------------------------------| +//! | 78 | `: bad configuration` | the settings, a secret file they name, the database or bucket name | +//! | 69 | `: unreachable` | no answer: refused, unknown host, timed out, or the server is starting or full | +//! | 77 | `: credentials refused` | wrong user, password or key, or a permission it needs is missing | +//! | 76 | `: TLS failed` | the TLS handshake failed: an unknown CA, or TLS the server doesn't offer | +//! | 1 | `: error` | anything else | //! -//! PostgreSQL errors are told apart fully. Other stores report bad -//! configuration (78) and otherwise 1. +//! The part is `data store` (at every start, and `--check`), `blob store` or +//! `coordinator` (`--check` only). PostgreSQL, S3 and NATS errors are told +//! apart; other stores report bad configuration and otherwise 1. use std::fmt; @@ -33,6 +34,32 @@ pub enum StartupFailure { Other, } +/// Which part of the data layer failed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Part { + DataStore, + BlobStore, + Coordinator, +} + +impl Part { + pub fn name(self) -> &'static str { + match self { + Part::DataStore => "data store", + Part::BlobStore => "blob store", + Part::Coordinator => "coordinator", + } + } + + fn what(self) -> &'static str { + match self { + Part::DataStore => "the database", + Part::BlobStore => "the blob store", + Part::Coordinator => "NATS", + } + } +} + impl StartupFailure { pub fn exit_code(self) -> i32 { match self { @@ -44,41 +71,66 @@ impl StartupFailure { } } + /// The case, in the words that follow the part's name. pub fn label(self) -> &'static str { match self { - StartupFailure::Config => "data store: bad configuration", - StartupFailure::Unreachable => "data store: unreachable", - StartupFailure::Credentials => "data store: credentials refused", - StartupFailure::Tls => "data store: TLS failed", - StartupFailure::Other => "data store: error", + StartupFailure::Config => "bad configuration", + StartupFailure::Unreachable => "unreachable", + StartupFailure::Credentials => "credentials refused", + StartupFailure::Tls => "TLS failed", + StartupFailure::Other => "error", } } /// What to check, in a sentence. - pub fn hint(self) -> &'static str { - match self { - StartupFailure::Config => { + pub fn hint(self, part: Part) -> String { + let what = part.what(); + match (self, part) { + (StartupFailure::Config, Part::DataStore) => { "Check the data store settings file, the secret files it names and the database name." + .into() } - StartupFailure::Unreachable => { - "Check that the database is running and that its host and port are reachable from here." + (StartupFailure::Config, Part::BlobStore) => { + "Check the blob store settings: the bucket, its region and endpoint, and the secret files they name." + .into() } - StartupFailure::Credentials => { - "Check the user name and password, and that the user may use the database." + (StartupFailure::Config, Part::Coordinator) => { + "Check the coordinator settings and the secret files they name.".into() } - StartupFailure::Tls => { - "Check that the database offers TLS and that its certificate is trusted here." + (StartupFailure::Unreachable, _) => format!( + "Check that {what} is running and that its host and port are reachable from here." + ), + (StartupFailure::Credentials, Part::BlobStore) => { + "Check the access key and secret, and that they may write and delete in the bucket." + .into() } - StartupFailure::Other => "", + (StartupFailure::Credentials, Part::DataStore) => { + "Check the user name and password, and that the user may use the database.".into() + } + (StartupFailure::Credentials, Part::Coordinator) => { + "Check the user name and password, or the token.".into() + } + (StartupFailure::Tls, _) => { + format!("Check that {what} offers TLS and that its certificate is trusted here.") + } + (StartupFailure::Other, _) => String::new(), } } - /// What a data store error at start-up means. + /// What a data store error means. pub fn of(err: &trc::Error) -> Self { + let code = err.value_as_str(trc::Key::Code); + let reason = err.value_as_str(trc::Key::Reason).unwrap_or_default(); if err.matches(trc::EventType::Store(trc::StoreEvent::PostgresqlError)) { - Self::of_postgres( - err.value_as_str(trc::Key::Code), - err.value_as_str(trc::Key::Reason).unwrap_or_default(), + Self::of_postgres(code, reason) + } else if err.matches(trc::EventType::Store(trc::StoreEvent::S3Error)) { + Self::of_s3( + err.value(trc::Key::Code).and_then(|v| match v { + trc::Value::UInt(n) => Some(*n), + trc::Value::Int(n) => u64::try_from(*n).ok(), + _ => None, + }), + reason, ) } else { StartupFailure::Other @@ -106,27 +158,76 @@ impl StartupFailure { None => StartupFailure::Other, } } + + /// `status` is the HTTP status when the service answered; otherwise + /// `reason` is the HTTP client's chain of errors. + fn of_s3(status: Option, reason: &str) -> Self { + match status { + // A key the service doesn't know, a bad signature, or a policy + // that refuses the operation + Some(401 | 403) => StartupFailure::Credentials, + // No such bucket, or a bucket in another region or endpoint + Some(301 | 307 | 400 | 404) => StartupFailure::Config, + Some(429 | 500..=599) => StartupFailure::Unreachable, + Some(_) => StartupFailure::Other, + None => { + let reason = reason.to_ascii_lowercase(); + if ["certificate", "tls", "ssl", "handshake"] + .iter() + .any(|w| reason.contains(w)) + { + StartupFailure::Tls + } else if [ + "connect", + "dns error", + "failed to lookup", + "timed out", + "connection refused", + "connection reset", + ] + .iter() + .any(|w| reason.contains(w)) + { + StartupFailure::Unreachable + } else { + StartupFailure::Other + } + } + } + } } #[derive(Debug)] pub struct StartupError { + pub part: Part, pub kind: StartupFailure, pub message: String, } impl StartupError { + /// A data store failure. pub fn new(kind: StartupFailure, message: impl Into) -> Self { + Self::of_part(Part::DataStore, kind, message) + } + + pub fn of_part(part: Part, kind: StartupFailure, message: impl Into) -> Self { StartupError { + part, kind, message: message.into(), } } + + /// "data store: unreachable" + pub fn headline(&self) -> String { + format!("{}: {}", self.part.name(), self.kind.label()) + } } impl fmt::Display for StartupError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}: {}", self.kind.label(), self.message)?; - let hint = self.kind.hint(); + write!(f, "{}: {}", self.headline(), self.message)?; + let hint = self.kind.hint(self.part); if !hint.is_empty() { write!(f, "\n {hint}")?; } @@ -136,7 +237,10 @@ impl fmt::Display for StartupError { #[cfg(test)] mod tests { - use super::StartupFailure::{self, *}; + use super::{ + Part, + StartupFailure::{self, *}, + }; #[test] fn postgres_cases() { @@ -208,15 +312,48 @@ mod tests { } #[test] - fn codes_are_distinct() { + fn s3_cases() { + let cases: &[(Option, &str, StartupFailure)] = &[ + (Some(403), "InvalidAccessKeyId", Credentials), + (Some(403), "SignatureDoesNotMatch", Credentials), + (Some(404), "NoSuchBucket", Config), + (Some(301), "PermanentRedirect", Config), + (Some(503), "SlowDown", Unreachable), + ( + None, + "error sending request: client error (Connect): tcp connect error: Connection refused (os error 111)", + Unreachable, + ), + ( + None, + "error sending request: client error (Connect): dns error: failed to lookup address information", + Unreachable, + ), + ( + None, + "error sending request: client error (Connect): invalid peer certificate: UnknownIssuer", + Tls, + ), + (Some(418), "", Other), + ]; + for (status, reason, want) in cases { + assert_eq!(StartupFailure::of_s3(*status, reason), *want, "{reason}"); + } + } + + #[test] + fn codes_and_words_are_distinct() { let all = [Config, Unreachable, Credentials, Tls, Other]; for a in all { for b in all { if a != b { assert_ne!(a.exit_code(), b.exit_code()); - assert!(!a.label().starts_with(b.label()), "{}", a.label()); + assert_ne!(a.label(), b.label()); } } + for part in [Part::DataStore, Part::BlobStore, Part::Coordinator] { + assert_eq!(a.hint(part).is_empty(), a == Other); + } } } }