--check: whether the data, blob store and coordinator can be used #473

Merged
jcoffey-dev merged 2 commits from feat/check-data-layer into main 2026-11-19 06:27:29 +00:00
6 changed files with 425 additions and 39 deletions

No files matched your search

+22 -1
View File
@@ -67,6 +67,7 @@ Options:
-u, --upgrade-store Upgrade the store's layout, then exit
--seed <PATH> Set up a new store from a file, then exit
--upgrade-plan Print what --upgrade-store would do, then exit
--check Check the data, blob store and coordinator, then exit
-h, --help Print help
-V, --version Print version
@@ -94,6 +95,14 @@ of JSON, the layout steps --upgrade-store would run, and exits: 0 when there
are none, 3 when there are and every one can run with older nodes still up,
4 when one needs every other node stopped first, 2 when a newer release has
changed the store, 1 when it can't tell (the message says why).
--check opens the data store, then tests the blob store (writing, reading
and deleting one small object) and the coordinator (connecting to NATS), and
prints a line for each. It starts nothing. It exits 0 when all three can be
used; otherwise the failing part and why are on its last lines, and the exit
code says which kind of failure: 78 bad configuration, 69 unreachable,
77 credentials refused, 76 TLS failed, 1 anything else. A server start that
can't open the data store exits the same way.
"#
);
@@ -108,6 +117,8 @@ enum StoreOp {
Seed(PathBuf),
// inbuxa: K-C9
UpgradePlan,
// inbuxa: K-C5
Check,
None,
}
@@ -174,6 +185,9 @@ impl BootManager {
("upgrade-plan", None) => {
import_export = StoreOp::UpgradePlan;
}
("check", None) => {
import_export = StoreOp::Check;
}
("seed", Some(value)) => {
import_export = StoreOp::Seed(value.into());
}
@@ -196,6 +210,11 @@ impl BootManager {
failed("Missing '--config' argument (or CONFIG_PATH) for this command.");
}
// inbuxa: K-C5: a check runs on its own and exits
if import_export == StoreOp::Check {
super::check::run(PathBuf::from(config_path.unwrap())).await;
}
// Initialize registry
let upgrade_store = import_export == StoreOp::UpgradeStore;
let seed = match &import_export {
@@ -386,6 +405,8 @@ impl BootManager {
core.queue_reindex(&imported).await;
std::process::exit(0);
}
// inbuxa: K-C5: ran and exited before the registry
StoreOp::Check => unreachable!(),
StoreOp::Console => {
// Store console
store_console(
@@ -432,7 +453,7 @@ pub fn build_ipc(has_pubsub: bool) -> (Ipc, IpcReceivers) {
fn startup_failed(err: store::registry::startup::StartupError) -> ! {
trc::event!(
Server(trc::ServerEvent::StartupError),
Details = err.kind.label(),
Details = err.headline(),
Reason = err.message.clone()
);
+144
View File
@@ -0,0 +1,144 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
//! inbuxa: K-C5: `--check`, whether the data layer this server is set up for
//! can be used: the data store, the blob store and the coordinator, in that
//! order. It reads the settings and writes one small object to the blob
//! store, which it deletes again; it starts nothing and claims no node id.
//!
//! Each part prints a line. A part that fails says why, in the words and with
//! the exit code of `store::registry::startup`; with more than one failing,
//! the exit code is the first one's. The Helm chart runs it as an init
//! container, so a pod that can't use the data layer says which part, and
//! why, in its status.
use coordinator::Coordinator;
use registry::schema::{enums::CompressionAlgo, structs};
use std::path::PathBuf;
use store::{
BlobStore, RegistryStore,
registry::{
bootstrap::Bootstrap,
startup::{Part, StartupError, StartupFailure},
},
};
pub async fn run(config_path: PathBuf) -> ! {
let registry = match RegistryStore::init(config_path.clone(), false).await {
Ok(registry) => registry,
Err(err) => fail(err),
};
if registry.is_bootstrap_mode() {
fail(StartupError::new(
StartupFailure::Config,
format!(
"there are no data store settings at {}",
config_path.display()
),
));
}
println!("ok data store");
let mut bp = Bootstrap::new(registry).await;
let mut failures = Vec::new();
match blob_store(&mut bp).await {
Ok(found) => println!("ok blob store: {found}"),
Err(err) => failures.push(err),
}
match coordinator(&mut bp).await {
Ok(found) => println!("ok coordinator: {found}"),
Err(err) => failures.push(err),
}
let code = failures.first().map(|err| err.kind.exit_code());
for err in failures {
eprintln!("⚠️ Check failed: {err}");
}
std::process::exit(code.unwrap_or(0));
}
fn fail(err: StartupError) -> ! {
eprintln!("⚠️ Check failed: {err}");
std::process::exit(err.kind.exit_code());
}
/// The setting couldn't be read: a fault of the data store, which answered
/// a moment ago.
fn unreadable(part: Part, err: trc::Error) -> StartupError {
StartupError::of_part(
Part::DataStore,
StartupFailure::of(&err),
format!("reading the {} settings: {err}", part.name()),
)
}
async fn blob_store(bp: &mut Bootstrap) -> Result<String, StartupError> {
let part = Part::BlobStore;
let config = bp
.setting::<structs::BlobStore>()
.await
.map_err(|err| unreadable(part, err))?;
if matches!(config, structs::BlobStore::Default) {
return Ok("kept in the data store".into());
}
let mut warnings = Vec::new();
let store = BlobStore::open(config, &bp.data_store, &mut warnings)
.await
.map_err(|err| StartupError::of_part(part, StartupFailure::Config, err))?;
// Everything the server does to a blob: write it, read it back, delete it
let key = format!(
".inbuxa-check/{}-{}",
bp.registry.local_hostname(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or_default()
);
let data = b"inbuxa --check";
let failed = |what: &str, err: trc::Error| {
StartupError::of_part(part, StartupFailure::of(&err), format!("{what}: {err}"))
};
store
.put_blob(key.as_bytes(), data, CompressionAlgo::None)
.await
.map_err(|err| failed("writing a test object", err))?;
let read = store
.get_blob(key.as_bytes(), 0..usize::MAX)
.await
.map_err(|err| failed("reading the test object back", err));
let deleted = store
.delete_blob(key.as_bytes())
.await
.map_err(|err| failed("deleting the test object", err));
match read? {
Some(read) if read == data => {}
_ => {
return Err(StartupError::of_part(
part,
StartupFailure::Other,
format!("the test object {key} read back differently, or not at all"),
));
}
}
deleted?;
Ok("wrote, read and deleted a test object".into())
}
async fn coordinator(bp: &mut Bootstrap) -> Result<String, StartupError> {
let part = Part::Coordinator;
let config = bp
.setting::<structs::Coordinator>()
.await
.map_err(|err| unreadable(part, err))?;
let none = matches!(config, structs::Coordinator::Disabled);
match Coordinator::probe(config).await {
Ok(Some(found)) => Ok(found),
Ok(None) if none => Ok("none, so this server can't run beside others".into()),
Ok(None) => Ok("not one this check can test".into()),
Err((kind, message)) => Err(StartupError::of_part(part, kind, message)),
}
}
+1
View File
@@ -19,6 +19,7 @@ pub mod application;
pub mod backup;
pub mod blob_moves; // inbuxa: blob-store-change spec, a blob store that moved
pub mod boot;
pub mod check; // inbuxa: K-C5, --check
pub mod compliance_roles; // inbuxa: personal-data catalog, the compliance roles
pub mod console;
pub mod defaults;
@@ -17,6 +17,7 @@ use std::{
use crate::Coordinator;
use async_nats::Client;
use registry::schema::structs::NatsCoordinator;
use store::registry::startup::StartupFailure;
use tokio::sync::watch;
use trc::ClusterEvent;
@@ -81,6 +82,70 @@ impl NatsPubSub {
.map_err(|err| format!("Failed to connect to Nats: {}", err))
}
/// inbuxa: K-C5: connects once, with no retrying in the background as
/// `open` does, and says why it couldn't. For `--check`.
pub async fn probe(config: NatsCoordinator) -> Result<String, (StartupFailure, String)> {
if config.addresses.is_empty() {
return Err((StartupFailure::Config, "no NATS addresses are set".into()));
}
let addresses = config.addresses.into_inner();
let named = addresses
.iter()
.map(|a| a.to_string())
.collect::<Vec<_>>()
.join(", ");
let timeout = config.timeout_connection.into_inner();
let mut opts = async_nats::ConnectOptions::new()
.connection_timeout(timeout)
.max_reconnects(Some(1))
.require_tls(config.use_tls);
let secret = |r: Result<_, String>| r.map_err(|err| (StartupFailure::Config, err));
if let (Some(user), Some(pass)) = (
config.auth_username,
secret(config.auth_secret.secret().await)?.map(|v| v.into_owned()),
) {
opts = opts.user_and_password(user.to_string(), pass.to_string());
} else if let Some(token) =
secret(config.credentials.secret().await)?.map(|v| v.into_owned())
{
opts = opts.token(token);
}
let connect = async {
let client = async_nats::connect_with_options(addresses, opts)
.await
.map_err(|err| {
use async_nats::ConnectErrorKind as K;
let kind = match err.kind() {
K::ServerParse => StartupFailure::Config,
K::Dns | K::Io | K::TimedOut | K::MaxReconnects => {
StartupFailure::Unreachable
}
K::Authentication | K::AuthorizationViolation => {
StartupFailure::Credentials
}
K::Tls => StartupFailure::Tls,
};
(kind, format!("{named}: {err}"))
})?;
client
.flush()
.await
.map_err(|err| (StartupFailure::Unreachable, format!("{named}: {err}")))?;
let server = client.server_info();
let _ = client.drain().await;
Ok(format!("NATS {} at {named}", server.version))
};
tokio::time::timeout(timeout * 2 + Duration::from_secs(5), connect)
.await
.unwrap_or_else(|_| {
Err((
StartupFailure::Unreachable,
format!("{named}: no answer in {}s", (timeout * 2).as_secs() + 5),
))
})
}
/// inbuxa: counts the client's reconnections. Core NATS delivers at
/// most once, so whatever the cluster published while this node was
/// away is gone; a watcher drops what those events would have changed.
+18
View File
@@ -2,6 +2,8 @@
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*
* Modified by Coffey Labs in 2026 for INBUXA.
*/
use crate::Coordinator;
@@ -66,6 +68,22 @@ impl Coordinator {
}
}
/// inbuxa: K-C5: whether the coordinator a setting names answers, for
/// `--check`. `Ok(None)` when there's none, or it's of a kind not
/// checked here.
pub async fn probe(
config: structs::Coordinator,
) -> Result<Option<String>, (store::registry::startup::StartupFailure, String)> {
match config {
structs::Coordinator::Disabled => Ok(None),
#[cfg(feature = "nats")]
structs::Coordinator::Nats(nats) => crate::backend::nats::NatsPubSub::probe(nats)
.await
.map(Some),
_ => Ok(None),
}
}
pub fn is_enabled(&self) -> bool {
!matches!(self, Coordinator::None)
}
+175 -38
View File
@@ -4,23 +4,24 @@
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
//! inbuxa: K-S6: why the data store couldn't be opened at start-up.
//! inbuxa: K-S6, K-C5: why a part of the data layer couldn't be used.
//!
//! Each case has its own exit code (from sysexits.h) and its own first words
//! on the error line, so a supervisor, a Kubernetes pre-flight check or a
//! person reading `kubectl describe` can tell them apart without parsing a
//! database driver's message:
//! Each case has its own exit code (from sysexits.h) and its own words on the
//! error line, after the part's name, so a supervisor, a Kubernetes pre-flight
//! check or a person reading `kubectl describe` can tell them apart without
//! parsing a driver's message:
//!
//! | exit | line starts | meaning |
//! |------|--------------------------------------------|-------------------------------------------|
//! | 78 | `data store: bad configuration` | the settings file, a secret file it names, or the database name |
//! | 69 | `data store: unreachable` | no answer: refused, unknown host, timed out, or the server is starting or full |
//! | 77 | `data store: credentials refused` | wrong user or password, or the role lacks a permission it needs |
//! | 76 | `data store: TLS failed` | the TLS handshake failed: an unknown CA, or TLS the server doesn't offer |
//! | 1 | `data store: error` | anything else |
//! | exit | line starts | meaning |
//! |------|---------------------------------|-------------------------------------------|
//! | 78 | `<part>: bad configuration` | the settings, a secret file they name, the database or bucket name |
//! | 69 | `<part>: unreachable` | no answer: refused, unknown host, timed out, or the server is starting or full |
//! | 77 | `<part>: credentials refused` | wrong user, password or key, or a permission it needs is missing |
//! | 76 | `<part>: TLS failed` | the TLS handshake failed: an unknown CA, or TLS the server doesn't offer |
//! | 1 | `<part>: error` | anything else |
//!
//! PostgreSQL errors are told apart fully. Other stores report bad
//! configuration (78) and otherwise 1.
//! The part is `data store` (at every start, and `--check`), `blob store` or
//! `coordinator` (`--check` only). PostgreSQL, S3 and NATS errors are told
//! apart; other stores report bad configuration and otherwise 1.
use std::fmt;
@@ -33,6 +34,32 @@ pub enum StartupFailure {
Other,
}
/// Which part of the data layer failed.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Part {
DataStore,
BlobStore,
Coordinator,
}
impl Part {
pub fn name(self) -> &'static str {
match self {
Part::DataStore => "data store",
Part::BlobStore => "blob store",
Part::Coordinator => "coordinator",
}
}
fn what(self) -> &'static str {
match self {
Part::DataStore => "the database",
Part::BlobStore => "the blob store",
Part::Coordinator => "NATS",
}
}
}
impl StartupFailure {
pub fn exit_code(self) -> i32 {
match self {
@@ -44,41 +71,66 @@ impl StartupFailure {
}
}
/// The case, in the words that follow the part's name.
pub fn label(self) -> &'static str {
match self {
StartupFailure::Config => "data store: bad configuration",
StartupFailure::Unreachable => "data store: unreachable",
StartupFailure::Credentials => "data store: credentials refused",
StartupFailure::Tls => "data store: TLS failed",
StartupFailure::Other => "data store: error",
StartupFailure::Config => "bad configuration",
StartupFailure::Unreachable => "unreachable",
StartupFailure::Credentials => "credentials refused",
StartupFailure::Tls => "TLS failed",
StartupFailure::Other => "error",
}
}
/// What to check, in a sentence.
pub fn hint(self) -> &'static str {
match self {
StartupFailure::Config => {
pub fn hint(self, part: Part) -> String {
let what = part.what();
match (self, part) {
(StartupFailure::Config, Part::DataStore) => {
"Check the data store settings file, the secret files it names and the database name."
.into()
}
StartupFailure::Unreachable => {
"Check that the database is running and that its host and port are reachable from here."
(StartupFailure::Config, Part::BlobStore) => {
"Check the blob store settings: the bucket, its region and endpoint, and the secret files they name."
.into()
}
StartupFailure::Credentials => {
"Check the user name and password, and that the user may use the database."
(StartupFailure::Config, Part::Coordinator) => {
"Check the coordinator settings and the secret files they name.".into()
}
StartupFailure::Tls => {
"Check that the database offers TLS and that its certificate is trusted here."
(StartupFailure::Unreachable, _) => format!(
"Check that {what} is running and that its host and port are reachable from here."
),
(StartupFailure::Credentials, Part::BlobStore) => {
"Check the access key and secret, and that they may write and delete in the bucket."
.into()
}
StartupFailure::Other => "",
(StartupFailure::Credentials, Part::DataStore) => {
"Check the user name and password, and that the user may use the database.".into()
}
(StartupFailure::Credentials, Part::Coordinator) => {
"Check the user name and password, or the token.".into()
}
(StartupFailure::Tls, _) => {
format!("Check that {what} offers TLS and that its certificate is trusted here.")
}
(StartupFailure::Other, _) => String::new(),
}
}
/// What a data store error at start-up means.
/// What a data store error means.
pub fn of(err: &trc::Error) -> Self {
let code = err.value_as_str(trc::Key::Code);
let reason = err.value_as_str(trc::Key::Reason).unwrap_or_default();
if err.matches(trc::EventType::Store(trc::StoreEvent::PostgresqlError)) {
Self::of_postgres(
err.value_as_str(trc::Key::Code),
err.value_as_str(trc::Key::Reason).unwrap_or_default(),
Self::of_postgres(code, reason)
} else if err.matches(trc::EventType::Store(trc::StoreEvent::S3Error)) {
Self::of_s3(
err.value(trc::Key::Code).and_then(|v| match v {
trc::Value::UInt(n) => Some(*n),
trc::Value::Int(n) => u64::try_from(*n).ok(),
_ => None,
}),
reason,
)
} else {
StartupFailure::Other
@@ -106,27 +158,76 @@ impl StartupFailure {
None => StartupFailure::Other,
}
}
/// `status` is the HTTP status when the service answered; otherwise
/// `reason` is the HTTP client's chain of errors.
fn of_s3(status: Option<u64>, reason: &str) -> Self {
match status {
// A key the service doesn't know, a bad signature, or a policy
// that refuses the operation
Some(401 | 403) => StartupFailure::Credentials,
// No such bucket, or a bucket in another region or endpoint
Some(301 | 307 | 400 | 404) => StartupFailure::Config,
Some(429 | 500..=599) => StartupFailure::Unreachable,
Some(_) => StartupFailure::Other,
None => {
let reason = reason.to_ascii_lowercase();
if ["certificate", "tls", "ssl", "handshake"]
.iter()
.any(|w| reason.contains(w))
{
StartupFailure::Tls
} else if [
"connect",
"dns error",
"failed to lookup",
"timed out",
"connection refused",
"connection reset",
]
.iter()
.any(|w| reason.contains(w))
{
StartupFailure::Unreachable
} else {
StartupFailure::Other
}
}
}
}
}
#[derive(Debug)]
pub struct StartupError {
pub part: Part,
pub kind: StartupFailure,
pub message: String,
}
impl StartupError {
/// A data store failure.
pub fn new(kind: StartupFailure, message: impl Into<String>) -> Self {
Self::of_part(Part::DataStore, kind, message)
}
pub fn of_part(part: Part, kind: StartupFailure, message: impl Into<String>) -> Self {
StartupError {
part,
kind,
message: message.into(),
}
}
/// "data store: unreachable"
pub fn headline(&self) -> String {
format!("{}: {}", self.part.name(), self.kind.label())
}
}
impl fmt::Display for StartupError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}: {}", self.kind.label(), self.message)?;
let hint = self.kind.hint();
write!(f, "{}: {}", self.headline(), self.message)?;
let hint = self.kind.hint(self.part);
if !hint.is_empty() {
write!(f, "\n {hint}")?;
}
@@ -136,7 +237,10 @@ impl fmt::Display for StartupError {
#[cfg(test)]
mod tests {
use super::StartupFailure::{self, *};
use super::{
Part,
StartupFailure::{self, *},
};
#[test]
fn postgres_cases() {
@@ -208,15 +312,48 @@ mod tests {
}
#[test]
fn codes_are_distinct() {
fn s3_cases() {
let cases: &[(Option<u64>, &str, StartupFailure)] = &[
(Some(403), "<Code>InvalidAccessKeyId</Code>", Credentials),
(Some(403), "<Code>SignatureDoesNotMatch</Code>", Credentials),
(Some(404), "<Code>NoSuchBucket</Code>", Config),
(Some(301), "<Code>PermanentRedirect</Code>", Config),
(Some(503), "<Code>SlowDown</Code>", Unreachable),
(
None,
"error sending request: client error (Connect): tcp connect error: Connection refused (os error 111)",
Unreachable,
),
(
None,
"error sending request: client error (Connect): dns error: failed to lookup address information",
Unreachable,
),
(
None,
"error sending request: client error (Connect): invalid peer certificate: UnknownIssuer",
Tls,
),
(Some(418), "", Other),
];
for (status, reason, want) in cases {
assert_eq!(StartupFailure::of_s3(*status, reason), *want, "{reason}");
}
}
#[test]
fn codes_and_words_are_distinct() {
let all = [Config, Unreachable, Credentials, Tls, Other];
for a in all {
for b in all {
if a != b {
assert_ne!(a.exit_code(), b.exit_code());
assert!(!a.label().starts_with(b.label()), "{}", a.label());
assert_ne!(a.label(), b.label());
}
}
for part in [Part::DataStore, Part::BlobStore, Part::Coordinator] {
assert_eq!(a.hint(part).is_empty(), a == Other);
}
}
}
}