From 122b7f7d2d7eb9a8ff7ba314b6b6390611ef637d Mon Sep 17 00:00:00 2001 From: John Coffey Date: Wed, 18 Nov 2026 08:42:33 -0700 Subject: [PATCH] --upgrade-store: upgrade the store layout, then exit For an orchestrator that upgrades the store as a job of its own before it rolls a release out (27.1's step to layout 3 can't roll; lab spike S5). It runs what a start runs on the store before serving, opens no ports and starts no services, and exits 0 when the store is at this build's layout (upgraded or found so), 1 when the upgrade failed, 2 when it refused. Unlike a starting node it doesn't wait out another node's lease: it refuses, and leaves that lease alone. A step that needs every other node stopped waits up to six minutes for them to count as gone, so pods stopped just before it don't fail the job, and is refused after that. The arguments are now read with CONFIG_PATH set too, an option that takes no value no longer swallows the next argument (so --upgrade-store --config works in that order), and a store command without a configuration exits 1 instead of 0. Closes #426 --- crates/common/src/manager/boot.rs | 163 +++++++++++++++++++----------- crates/main/src/main.rs | 52 ++++++++++ crates/migration/src/layout.rs | 119 ++++++++++++++++++---- crates/migration/src/lib.rs | 17 +++- tests/src/store_version.rs | 69 +++++++++++-- 5 files changed, 331 insertions(+), 89 deletions(-) diff --git a/crates/common/src/manager/boot.rs b/crates/common/src/manager/boot.rs index d6908d64..3f612859 100644 --- a/crates/common/src/manager/boot.rs +++ b/crates/common/src/manager/boot.rs @@ -30,6 +30,9 @@ pub struct BootManager { pub inner: Arc, pub servers: Listeners, pub ipc_rxs: IpcReceivers, + /// inbuxa: K-S3: started with `--upgrade-store`: no listeners, no + /// services; main upgrades the store's layout and exits + pub upgrade_store: bool, } pub struct IpcReceivers { @@ -52,6 +55,7 @@ Options: -e, --export Export all store data to a specific path -i, --import Import store data from a specific path -o, --console Open the store console + -u, --upgrade-store Upgrade the store's layout, then exit -h, --help Print help -V, --version Print version @@ -61,6 +65,12 @@ index, which belongs to one search backend. An import into an empty store queues the index to be rebuilt when the server next starts. EXPORT_TYPES limits an export to some of: data, registry, blob, changelog, queue, report, telemetry, tasks. + +--upgrade-store does what a start does to the store before it serves, and +exits: 0 when the store is at this release's layout (whether it upgraded it +or found it there), 1 when the upgrade failed, 2 when it refused because +another node holds the upgrade lease, or still runs when a step needs every +other node stopped. It opens no ports and starts no services. "# ); @@ -69,6 +79,8 @@ enum StoreOp { Export(BackupParams), Import(PathBuf), Console, + // inbuxa: K-S3 + UpgradeStore, None, } @@ -77,67 +89,88 @@ impl BootManager { let mut config_path = std::env::var("CONFIG_PATH").ok(); let mut import_export = StoreOp::None; - if config_path.is_none() { - let mut args = std::env::args().skip(1); + // inbuxa: K-S3: the arguments are read with CONFIG_PATH set too, so + // `--upgrade-store` works where the path comes from the environment; + // and an option that takes no value no longer swallows the next one, + // so `--upgrade-store --config ` works in that order + let mut args = std::env::args().skip(1); + let has_args = args.len() > 0; + while let Some(arg) = args.next().and_then(|arg| { + arg.strip_prefix("--") + .or_else(|| arg.strip_prefix('-')) + .map(|arg| arg.to_string()) + }) { + let (key, inline) = match arg.split_once('=') { + Some((key, value)) => (key.to_string(), Some(value.trim().to_string())), + None => (arg, None), + }; + let takes_value = matches!( + key.as_str(), + "config" | "c" | "export" | "e" | "import" | "i" + ); + let value = if takes_value { + inline.or_else(|| args.next()) + } else { + inline + }; - while let Some(arg) = args.next().and_then(|arg| { - arg.strip_prefix("--") - .or_else(|| arg.strip_prefix('-')) - .map(|arg| arg.to_string()) - }) { - let (key, value) = if let Some((key, value)) = arg.split_once('=') { - (key.to_string(), Some(value.trim().to_string())) - } else { - (arg, args.next()) - }; - - match (key.as_str(), value) { - ("help" | "h", _) => { - eprintln!("{HELP}"); - std::process::exit(0); - } - ("version" | "V", _) => { - println!("{}", types::brand_version!()); - std::process::exit(0); - } - ("config" | "c", Some(value)) => { - config_path = Some(value); - } - ("export" | "e", Some(value)) => { - import_export = StoreOp::Export(BackupParams::new(value.into())); - } - ("import" | "i", Some(value)) => { - import_export = StoreOp::Import(value.into()); - } - ("console" | "o", None) => { - import_export = StoreOp::Console; - } - (_, None) => { - failed(&format!("Unrecognized command '{key}', try '--help'.")); - } - (_, Some(_)) => failed(&format!( - "Missing value for argument '{key}', try '--help'." - )), - } - } - - if config_path.is_none() { - if import_export == StoreOp::None { + match (key.as_str(), value) { + ("help" | "h", _) => { eprintln!("{HELP}"); - } else { - eprintln!("Missing '--config' argument for import/export.") + std::process::exit(0); } - std::process::exit(0); + ("version" | "V", _) => { + println!("{}", types::brand_version!()); + std::process::exit(0); + } + ("config" | "c", Some(value)) => { + config_path = Some(value); + } + ("export" | "e", Some(value)) => { + import_export = StoreOp::Export(BackupParams::new(value.into())); + } + ("import" | "i", Some(value)) => { + import_export = StoreOp::Import(value.into()); + } + ("console" | "o", None) => { + import_export = StoreOp::Console; + } + ("upgrade-store" | "u", None) => { + import_export = StoreOp::UpgradeStore; + } + (_, None) if takes_value => failed(&format!( + "Missing value for argument '{key}', try '--help'." + )), + (_, None) => { + failed(&format!("Unrecognized command '{key}', try '--help'.")); + } + (_, Some(_)) => failed(&format!("Argument '{key}' takes no value, try '--help'.")), } } + if config_path.is_none() { + if import_export == StoreOp::None { + eprintln!("{HELP}"); + std::process::exit(if has_args { 1 } else { 0 }); + } + // A job that asked for a store operation must not pass without one + failed("Missing '--config' argument (or CONFIG_PATH) for this command."); + } + // Initialize registry + let upgrade_store = import_export == StoreOp::UpgradeStore; let registry = RegistryStore::init( PathBuf::from(config_path.unwrap()), - import_export == StoreOp::None, + import_export == StoreOp::None || upgrade_store, ) .await .failed("⚠️ Startup failed"); + if upgrade_store && (registry.is_bootstrap_mode() || registry.is_recovery_mode()) { + failed( + "--upgrade-store needs a configured server: there is no data store to \ + upgrade in bootstrap mode, and recovery mode is for a person at the console.", + ); + } let mut bootstrap = Bootstrap::new(registry).await; // Add safe defaults if missing @@ -146,8 +179,14 @@ impl BootManager { } // Start listeners - let mut servers = Listeners::parse(&mut bootstrap).await; - servers.bind_and_drop_priv(&mut bootstrap); + // inbuxa: K-S3: not for --upgrade-store, which serves nothing + let mut servers = if upgrade_store { + Listeners::default() + } else { + let servers = Listeners::parse(&mut bootstrap).await; + servers.bind_and_drop_priv(&mut bootstrap); + servers + }; // Parse storage let storage = Storage::parse(&mut bootstrap).await; @@ -156,7 +195,7 @@ impl BootManager { let telemetry = Telemetry::parse(&mut bootstrap, &storage).await; match import_export { - StoreOp::None => { + StoreOp::None | StoreOp::UpgradeStore => { // Parse components let core: Box = Box::new(Box::pin(Core::parse(&mut bootstrap, storage)).await); @@ -167,7 +206,14 @@ impl BootManager { telemetry.enable(); - if bootstrap.registry.is_bootstrap_mode() { + if upgrade_store { + trc::event!( + Server(trc::ServerEvent::Startup), + Details = "Started with --upgrade-store: upgrading the store layout, then exiting", + Hostname = bootstrap.registry.local_hostname().to_string(), + Version = types::brand_version!(), + ); + } else if bootstrap.registry.is_bootstrap_mode() { trc::event!( Server(trc::ServerEvent::BootstrapMode), Hostname = bootstrap.registry.local_hostname().to_string(), @@ -216,7 +262,7 @@ impl BootManager { cache, }); - if !bootstrap.registry.is_recovery_mode() { + if !bootstrap.registry.is_recovery_mode() && !upgrade_store { // Load spam model if let Err(err) = inner.build_server().spam_model_reload().await { trc::error!( @@ -235,9 +281,11 @@ impl BootManager { } // Parse TCP acceptors - servers - .parse_tcp_acceptors(&mut bootstrap, inner.clone()) - .await; + if !upgrade_store { + servers + .parse_tcp_acceptors(&mut bootstrap, inner.clone()) + .await; + } // inbuxa: a reload isn't refused over objects that failed here inner.build_server().record_build_errors(&bootstrap.errors); @@ -251,6 +299,7 @@ impl BootManager { bootstrap, servers, ipc_rxs, + upgrade_store, } } StoreOp::Export(path) => { diff --git a/crates/main/src/main.rs b/crates/main/src/main.rs index f7eba759..c20a3073 100644 --- a/crates/main/src/main.rs +++ b/crates/main/src/main.rs @@ -60,6 +60,11 @@ async fn main() -> std::io::Result<()> { // Load config and apply macros let mut init = Box::pin(BootManager::init()).await; + // inbuxa: K-S3: --upgrade-store upgrades the store's layout and exits + if init.upgrade_store { + std::process::exit(upgrade_store_and_exit(&init.inner.build_server()).await); + } + // Migrate database if let Err(err) = migration::try_migrate(&init.inner.build_server()).await { trc::event!( @@ -186,3 +191,50 @@ fn spawn_listener( ), } } + +/// inbuxa: K-S3: the exit code of `--upgrade-store`: 0 when the store is at +/// this build's layout (upgraded now or already), 1 when the upgrade failed, +/// 2 when it was refused because another node is in the way. +async fn upgrade_store_and_exit(server: &common::Server) -> i32 { + use migration::layout::Outcome; + + let (code, message) = match migration::upgrade_store(server).await { + Ok(Outcome::Current) => ( + 0, + format!( + "The store layout is at version {}: nothing to upgrade.", + common::INBUXA_STORE_VERSION + ), + ), + Ok(Outcome::Upgraded { from }) => ( + 0, + format!( + "Upgraded the store layout from version {from} to {}.", + common::INBUXA_STORE_VERSION + ), + ), + Ok(Outcome::Refused(reason)) => { + trc::event!( + Server(trc::ServerEvent::StartupError), + Details = "Refused to upgrade the store layout", + Reason = reason.clone(), + ); + (2, format!("Refused to upgrade the store layout: {reason}.")) + } + Err(err) => { + let message = format!("Failed to upgrade the store layout: {err}"); + trc::event!( + Server(trc::ServerEvent::StartupError), + Details = "Failed to upgrade the store layout", + Reason = err, + ); + (1, message) + } + }; + eprintln!("{message}"); + + // Let the log reach its collector + Collector::shutdown(); + tokio::time::sleep(Duration::from_secs(1)).await; + code +} diff --git a/crates/migration/src/layout.rs b/crates/migration/src/layout.rs index f64c45e2..4ba97626 100644 --- a/crates/migration/src/layout.rs +++ b/crates/migration/src/layout.rs @@ -36,6 +36,33 @@ const LEASE_RENEWAL: Duration = Duration::from_secs(60); const WAIT_INTERVAL: Duration = Duration::from_secs(10); const WATCH_INTERVAL: Duration = Duration::from_secs(60); const UPDATING_DOCS: &str = "https://docs.inbuxa.org/install/updating/"; +/// K-S3: how long `--upgrade-store` waits for other nodes to show as +/// stopped before a step that needs them stopped. A node shows as stopped a +/// few minutes after its last heartbeat, so a pod that was just scaled down +/// still counts for a while. +const UPGRADE_ONLY_NODE_WAIT: Duration = Duration::from_secs(6 * 60); + +/// inbuxa: K-S3: who is running the steps. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Run { + /// A node starting: it waits for whatever is in the way, then serves. + Boot, + /// `--upgrade-store`: it upgrades and exits, and refuses rather than + /// waits for another node's upgrade. + UpgradeOnly, +} + +/// What a run of the steps came to. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Outcome { + /// The store was at this build's layout already, or another node + /// brought it there while this one waited. + Current, + /// This node upgraded it, from this version. + Upgraded { from: u32 }, + /// `--upgrade-store` refused, for this reason. + Refused(String), +} /// One step, from the version it starts at to the next. struct Step { @@ -103,27 +130,39 @@ pub async fn upgrade_layout( server: &Server, store_version: Option, new_install: bool, -) -> trc::Result<()> { + run: Run, +) -> trc::Result { let version = store_version.unwrap_or(if new_install { INBUXA_STORE_VERSION } else { 1 }); - if version < INBUXA_STORE_VERSION { - run_steps(server, server.registry().node_id()).await?; - } else if store_version.is_none() { - write_mark(server.store(), INBUXA_STORE_VERSION).await?; - } + let outcome = if version < INBUXA_STORE_VERSION { + run_steps_as(server, server.registry().node_id(), run).await? + } else { + if store_version.is_none() { + write_mark(server.store(), INBUXA_STORE_VERSION).await?; + } + Outcome::Current + }; // SL-7 - trc::event!( - Server(trc::ServerEvent::Startup), - Details = format!("Store layout version {INBUXA_STORE_VERSION}"), - ); + if !matches!(outcome, Outcome::Refused(_)) { + trc::event!( + Server(trc::ServerEvent::Startup), + Details = format!("Store layout version {INBUXA_STORE_VERSION}"), + ); + } - Ok(()) + Ok(outcome) } /// Runs every step from the store's version up to this build's, as node /// `node_id`, or waits for the node that runs them (SL-3). pub async fn run_steps(server: &Server, node_id: u16) -> trc::Result<()> { + run_steps_as(server, node_id, Run::Boot).await.map(|_| ()) +} + +/// As [`run_steps`]; under [`Run::UpgradeOnly`] another node's lease is a +/// refusal rather than something to wait out. +pub async fn run_steps_as(server: &Server, node_id: u16, run: Run) -> trc::Result { let store = server.store(); let mut waiting = false; @@ -137,15 +176,26 @@ pub async fn run_steps(server: &Server, node_id: u16) -> trc::Result<()> { Version = version, ); } - return Ok(()); + return Ok(Outcome::Current); } if let Some(lease) = Lease::take(store, node_id).await? { - let result = run_held(server, &lease, version).await; + let result = run_held(server, &lease, version, run).await; lease.release(store).await; return result; } + if run == Run::UpgradeOnly { + return Ok(Outcome::Refused(match Lease::holder(store).await? { + Some((holder, expires)) => format!( + "node {holder} holds the store layout lease and is upgrading the store; \ + it runs out in {}s unless that node renews it", + expires.saturating_sub(now()) + ), + None => "another node took the store layout lease first".to_string(), + })); + } + if !waiting { trc::event!( Server(trc::ServerEvent::Startup), @@ -158,10 +208,14 @@ pub async fn run_steps(server: &Server, node_id: u16) -> trc::Result<()> { } } -async fn run_held(server: &Server, lease: &Lease, version: u32) -> trc::Result<()> { +async fn run_held(server: &Server, lease: &Lease, version: u32, run: Run) -> trc::Result { let store = server.store(); // Another node may have run a step between the read and the lease let mut version = store_version(store).await?.unwrap_or(version); + let from = version; + if version >= INBUXA_STORE_VERSION { + return Ok(Outcome::Current); + } while version < INBUXA_STORE_VERSION { let Some(step) = STEPS.iter().find(|step| step.from == version) else { @@ -169,8 +223,14 @@ async fn run_held(server: &Server, lease: &Lease, version: u32) -> trc::Result<( "The data store is at layout version {version}, and this release has no step from it" )); }; - if !step.rolling_safe { - wait_for_other_nodes(server).await?; + if !step.rolling_safe + && let Some(running) = wait_for_other_nodes(server, run).await? + { + return Ok(Outcome::Refused(format!( + "the step from layout version {version} needs every other node stopped, \ + and these still show as running: {}", + running.join(", ") + ))); } // SL-8 @@ -206,16 +266,19 @@ async fn run_held(server: &Server, lease: &Lease, version: u32) -> trc::Result<( ); } - Ok(()) + Ok(Outcome::Upgraded { from }) } /// A step that isn't rolling-safe waits until no other node shows as active /// (SL-6). A single node never waits: only other node ids on other hosts -/// count, so this node's own earlier lease doesn't. -async fn wait_for_other_nodes(server: &Server) -> trc::Result<()> { +/// count, so this node's own earlier lease doesn't. Under +/// [`Run::UpgradeOnly`] it gives up after [`UPGRADE_ONLY_NODE_WAIT`] and +/// returns the nodes still running. +async fn wait_for_other_nodes(server: &Server, run: Run) -> trc::Result>> { let me = server.registry().node_id() as u64; let hostname = server.registry().local_hostname().to_string(); let mut logged: Option = None; + let started = Instant::now(); loop { let others = server @@ -232,7 +295,10 @@ async fn wait_for_other_nodes(server: &Server) -> trc::Result<()> { .map(|node| node.hostname) .collect::>(); if others.is_empty() { - return Ok(()); + return Ok(None); + } + if run == Run::UpgradeOnly && started.elapsed() >= UPGRADE_ONLY_NODE_WAIT { + return Ok(Some(others)); } if logged.is_none_or(|at| at.elapsed() >= Duration::from_secs(60)) { trc::event!( @@ -302,6 +368,19 @@ async fn swap_lease(store: &Store, current: Option<&[u8]>, new: Vec) -> trc: } impl Lease { + /// Who holds the lease and until when, if anyone does and it hasn't run out. + async fn holder(store: &Store) -> trc::Result> { + Ok(store + .get_value::(ValueKey::from(lease_class())) + .await + .caused_by(trc::location!())? + .and_then(|Raw(bytes)| { + let holder = bytes.as_slice().deserialize_be_u16(0).ok()?; + let expires = bytes.as_slice().deserialize_be_u64(2).ok()?; + (expires > now()).then_some((holder, expires)) + })) + } + /// Takes the lease if nobody holds it, it ran out, or it's this node's /// own (left by a run that stopped). async fn take(store: &Store, node_id: u16) -> trc::Result> { diff --git a/crates/migration/src/lib.rs b/crates/migration/src/lib.rs index f8b54a75..b5ea4405 100644 --- a/crates/migration/src/lib.rs +++ b/crates/migration/src/lib.rs @@ -23,6 +23,17 @@ pub mod layout; pub mod v016; pub async fn try_migrate(server: &Server) -> trc::Result<()> { + migrate(server, layout::Run::Boot).await.map(|_| ()) +} + +/// inbuxa: K-S3: `--upgrade-store`. What a start would do to the store +/// before serving, and nothing else; another node's upgrade is refused +/// rather than waited for. +pub async fn upgrade_store(server: &Server) -> trc::Result { + migrate(server, layout::Run::UpgradeOnly).await +} + +async fn migrate(server: &Server, run: layout::Run) -> trc::Result { // inbuxa: a store a newer release has changed is refused before anything // below writes to it. let store_version = layout::store_version(server.store()).await?; @@ -47,7 +58,7 @@ pub async fn try_migrate(server: &Server) -> trc::Result<()> { { Some(DATABASE_SCHEMA_VERSION) => { if !std::env::var("DANGER_FORCE_MIGRATE").is_ok_and(|v| v == "1") { - return layout::upgrade_layout(server, store_version, false).await; + return layout::upgrade_layout(server, store_version, false, run).await; } } Some(0..=4) => { @@ -74,7 +85,7 @@ pub async fn try_migrate(server: &Server) -> trc::Result<()> { _ => { if is_new_install(server).await.caused_by(trc::location!())? { write_schema_version(server).await?; - return layout::upgrade_layout(server, store_version, true).await; + return layout::upgrade_layout(server, store_version, true, run).await; } else { abort(concat!( "You must first upgrade to version 0.15, please read ", @@ -86,7 +97,7 @@ pub async fn try_migrate(server: &Server) -> trc::Result<()> { migrate_v0_16(server).await?; write_schema_version(server).await?; - layout::upgrade_layout(server, store_version, false).await + layout::upgrade_layout(server, store_version, false, run).await } async fn write_schema_version(server: &Server) -> trc::Result<()> { diff --git a/tests/src/store_version.rs b/tests/src/store_version.rs index c429e13f..12b41f64 100644 --- a/tests/src/store_version.rs +++ b/tests/src/store_version.rs @@ -94,6 +94,7 @@ async fn store_version() { layout_step_1_2(server).await; layout_lease(server).await; + upgrade_store_only(server).await; layout_step_2_3(server).await; } @@ -245,15 +246,14 @@ async fn write_lease(server: &Server, node_id: u16, expires_at: u64) { } async fn lease_held(server: &Server) -> bool { - server - .store() - .get_value::(AnyKey { - subspace: SUBSPACE_INBUXA, - key: migration::layout::LAYOUT_LEASE_KEY.to_vec(), - }) - .await - .unwrap() - .is_some() + // A lease is a node id and an expiry, so it's read as it is + get_raw( + server.store(), + SUBSPACE_INBUXA, + migration::layout::LAYOUT_LEASE_KEY, + ) + .await + .is_some() } // SL-3: a node that can't take the lease waits for the mark; a lease that @@ -309,6 +309,57 @@ async fn layout_lease(server: &Server) { assert!(!lease_held(server).await, "lease not released"); } +// inbuxa: K-S3: `--upgrade-store` upgrades and stops; it changes nothing on +// a current store, and refuses, rather than waits, while another node holds +// the lease. +async fn upgrade_store_only(server: &Server) { + use migration::layout::Outcome; + + // Current: nothing to do, twice over + write_mark(server, Some(INBUXA_STORE_VERSION)).await; + for _ in 0..2 { + assert_eq!( + migration::upgrade_store(server).await.unwrap(), + Outcome::Current + ); + assert_eq!(read_mark(server).await, Some(INBUXA_STORE_VERSION)); + } + + // Another node holds the lease: refused at once, and its lease is left + write_mark(server, Some(1)).await; + write_lease(server, 900, now() + 600).await; + let outcome = tokio::time::timeout(Duration::from_secs(5), migration::upgrade_store(server)) + .await + .expect("waited on another node's lease instead of refusing") + .unwrap(); + match outcome { + Outcome::Refused(reason) => assert!(reason.contains("node 900"), "{reason}"), + other => panic!("expected a refusal, got {other:?}"), + } + assert_eq!( + read_mark(server).await, + Some(1), + "upgraded under another lease" + ); + assert!( + lease_held(server).await, + "the other node's lease was removed" + ); + + // Once it's gone: upgraded, lease released, and a second run is a no-op + write_lease(server, 900, now() - 1).await; + assert_eq!( + migration::upgrade_store(server).await.unwrap(), + Outcome::Upgraded { from: 1 } + ); + assert_eq!(read_mark(server).await, Some(INBUXA_STORE_VERSION)); + assert!(!lease_held(server).await, "lease not released"); + assert_eq!( + migration::upgrade_store(server).await.unwrap(), + Outcome::Current + ); +} + async fn applications(server: &Server) -> Vec<(String, bool)> { let mut apps = server .registry() -- 2.54.0