diff --git a/crates/common/src/manager/boot.rs b/crates/common/src/manager/boot.rs index d6908d64..3f612859 100644 --- a/crates/common/src/manager/boot.rs +++ b/crates/common/src/manager/boot.rs @@ -30,6 +30,9 @@ pub struct BootManager { pub inner: Arc, pub servers: Listeners, pub ipc_rxs: IpcReceivers, + /// inbuxa: K-S3: started with `--upgrade-store`: no listeners, no + /// services; main upgrades the store's layout and exits + pub upgrade_store: bool, } pub struct IpcReceivers { @@ -52,6 +55,7 @@ Options: -e, --export Export all store data to a specific path -i, --import Import store data from a specific path -o, --console Open the store console + -u, --upgrade-store Upgrade the store's layout, then exit -h, --help Print help -V, --version Print version @@ -61,6 +65,12 @@ index, which belongs to one search backend. An import into an empty store queues the index to be rebuilt when the server next starts. EXPORT_TYPES limits an export to some of: data, registry, blob, changelog, queue, report, telemetry, tasks. + +--upgrade-store does what a start does to the store before it serves, and +exits: 0 when the store is at this release's layout (whether it upgraded it +or found it there), 1 when the upgrade failed, 2 when it refused because +another node holds the upgrade lease, or still runs when a step needs every +other node stopped. It opens no ports and starts no services. "# ); @@ -69,6 +79,8 @@ enum StoreOp { Export(BackupParams), Import(PathBuf), Console, + // inbuxa: K-S3 + UpgradeStore, None, } @@ -77,67 +89,88 @@ impl BootManager { let mut config_path = std::env::var("CONFIG_PATH").ok(); let mut import_export = StoreOp::None; - if config_path.is_none() { - let mut args = std::env::args().skip(1); + // inbuxa: K-S3: the arguments are read with CONFIG_PATH set too, so + // `--upgrade-store` works where the path comes from the environment; + // and an option that takes no value no longer swallows the next one, + // so `--upgrade-store --config ` works in that order + let mut args = std::env::args().skip(1); + let has_args = args.len() > 0; + while let Some(arg) = args.next().and_then(|arg| { + arg.strip_prefix("--") + .or_else(|| arg.strip_prefix('-')) + .map(|arg| arg.to_string()) + }) { + let (key, inline) = match arg.split_once('=') { + Some((key, value)) => (key.to_string(), Some(value.trim().to_string())), + None => (arg, None), + }; + let takes_value = matches!( + key.as_str(), + "config" | "c" | "export" | "e" | "import" | "i" + ); + let value = if takes_value { + inline.or_else(|| args.next()) + } else { + inline + }; - while let Some(arg) = args.next().and_then(|arg| { - arg.strip_prefix("--") - .or_else(|| arg.strip_prefix('-')) - .map(|arg| arg.to_string()) - }) { - let (key, value) = if let Some((key, value)) = arg.split_once('=') { - (key.to_string(), Some(value.trim().to_string())) - } else { - (arg, args.next()) - }; - - match (key.as_str(), value) { - ("help" | "h", _) => { - eprintln!("{HELP}"); - std::process::exit(0); - } - ("version" | "V", _) => { - println!("{}", types::brand_version!()); - std::process::exit(0); - } - ("config" | "c", Some(value)) => { - config_path = Some(value); - } - ("export" | "e", Some(value)) => { - import_export = StoreOp::Export(BackupParams::new(value.into())); - } - ("import" | "i", Some(value)) => { - import_export = StoreOp::Import(value.into()); - } - ("console" | "o", None) => { - import_export = StoreOp::Console; - } - (_, None) => { - failed(&format!("Unrecognized command '{key}', try '--help'.")); - } - (_, Some(_)) => failed(&format!( - "Missing value for argument '{key}', try '--help'." - )), - } - } - - if config_path.is_none() { - if import_export == StoreOp::None { + match (key.as_str(), value) { + ("help" | "h", _) => { eprintln!("{HELP}"); - } else { - eprintln!("Missing '--config' argument for import/export.") + std::process::exit(0); } - std::process::exit(0); + ("version" | "V", _) => { + println!("{}", types::brand_version!()); + std::process::exit(0); + } + ("config" | "c", Some(value)) => { + config_path = Some(value); + } + ("export" | "e", Some(value)) => { + import_export = StoreOp::Export(BackupParams::new(value.into())); + } + ("import" | "i", Some(value)) => { + import_export = StoreOp::Import(value.into()); + } + ("console" | "o", None) => { + import_export = StoreOp::Console; + } + ("upgrade-store" | "u", None) => { + import_export = StoreOp::UpgradeStore; + } + (_, None) if takes_value => failed(&format!( + "Missing value for argument '{key}', try '--help'." + )), + (_, None) => { + failed(&format!("Unrecognized command '{key}', try '--help'.")); + } + (_, Some(_)) => failed(&format!("Argument '{key}' takes no value, try '--help'.")), } } + if config_path.is_none() { + if import_export == StoreOp::None { + eprintln!("{HELP}"); + std::process::exit(if has_args { 1 } else { 0 }); + } + // A job that asked for a store operation must not pass without one + failed("Missing '--config' argument (or CONFIG_PATH) for this command."); + } + // Initialize registry + let upgrade_store = import_export == StoreOp::UpgradeStore; let registry = RegistryStore::init( PathBuf::from(config_path.unwrap()), - import_export == StoreOp::None, + import_export == StoreOp::None || upgrade_store, ) .await .failed("⚠️ Startup failed"); + if upgrade_store && (registry.is_bootstrap_mode() || registry.is_recovery_mode()) { + failed( + "--upgrade-store needs a configured server: there is no data store to \ + upgrade in bootstrap mode, and recovery mode is for a person at the console.", + ); + } let mut bootstrap = Bootstrap::new(registry).await; // Add safe defaults if missing @@ -146,8 +179,14 @@ impl BootManager { } // Start listeners - let mut servers = Listeners::parse(&mut bootstrap).await; - servers.bind_and_drop_priv(&mut bootstrap); + // inbuxa: K-S3: not for --upgrade-store, which serves nothing + let mut servers = if upgrade_store { + Listeners::default() + } else { + let servers = Listeners::parse(&mut bootstrap).await; + servers.bind_and_drop_priv(&mut bootstrap); + servers + }; // Parse storage let storage = Storage::parse(&mut bootstrap).await; @@ -156,7 +195,7 @@ impl BootManager { let telemetry = Telemetry::parse(&mut bootstrap, &storage).await; match import_export { - StoreOp::None => { + StoreOp::None | StoreOp::UpgradeStore => { // Parse components let core: Box = Box::new(Box::pin(Core::parse(&mut bootstrap, storage)).await); @@ -167,7 +206,14 @@ impl BootManager { telemetry.enable(); - if bootstrap.registry.is_bootstrap_mode() { + if upgrade_store { + trc::event!( + Server(trc::ServerEvent::Startup), + Details = "Started with --upgrade-store: upgrading the store layout, then exiting", + Hostname = bootstrap.registry.local_hostname().to_string(), + Version = types::brand_version!(), + ); + } else if bootstrap.registry.is_bootstrap_mode() { trc::event!( Server(trc::ServerEvent::BootstrapMode), Hostname = bootstrap.registry.local_hostname().to_string(), @@ -216,7 +262,7 @@ impl BootManager { cache, }); - if !bootstrap.registry.is_recovery_mode() { + if !bootstrap.registry.is_recovery_mode() && !upgrade_store { // Load spam model if let Err(err) = inner.build_server().spam_model_reload().await { trc::error!( @@ -235,9 +281,11 @@ impl BootManager { } // Parse TCP acceptors - servers - .parse_tcp_acceptors(&mut bootstrap, inner.clone()) - .await; + if !upgrade_store { + servers + .parse_tcp_acceptors(&mut bootstrap, inner.clone()) + .await; + } // inbuxa: a reload isn't refused over objects that failed here inner.build_server().record_build_errors(&bootstrap.errors); @@ -251,6 +299,7 @@ impl BootManager { bootstrap, servers, ipc_rxs, + upgrade_store, } } StoreOp::Export(path) => { diff --git a/crates/main/src/main.rs b/crates/main/src/main.rs index f7eba759..c20a3073 100644 --- a/crates/main/src/main.rs +++ b/crates/main/src/main.rs @@ -60,6 +60,11 @@ async fn main() -> std::io::Result<()> { // Load config and apply macros let mut init = Box::pin(BootManager::init()).await; + // inbuxa: K-S3: --upgrade-store upgrades the store's layout and exits + if init.upgrade_store { + std::process::exit(upgrade_store_and_exit(&init.inner.build_server()).await); + } + // Migrate database if let Err(err) = migration::try_migrate(&init.inner.build_server()).await { trc::event!( @@ -186,3 +191,50 @@ fn spawn_listener( ), } } + +/// inbuxa: K-S3: the exit code of `--upgrade-store`: 0 when the store is at +/// this build's layout (upgraded now or already), 1 when the upgrade failed, +/// 2 when it was refused because another node is in the way. +async fn upgrade_store_and_exit(server: &common::Server) -> i32 { + use migration::layout::Outcome; + + let (code, message) = match migration::upgrade_store(server).await { + Ok(Outcome::Current) => ( + 0, + format!( + "The store layout is at version {}: nothing to upgrade.", + common::INBUXA_STORE_VERSION + ), + ), + Ok(Outcome::Upgraded { from }) => ( + 0, + format!( + "Upgraded the store layout from version {from} to {}.", + common::INBUXA_STORE_VERSION + ), + ), + Ok(Outcome::Refused(reason)) => { + trc::event!( + Server(trc::ServerEvent::StartupError), + Details = "Refused to upgrade the store layout", + Reason = reason.clone(), + ); + (2, format!("Refused to upgrade the store layout: {reason}.")) + } + Err(err) => { + let message = format!("Failed to upgrade the store layout: {err}"); + trc::event!( + Server(trc::ServerEvent::StartupError), + Details = "Failed to upgrade the store layout", + Reason = err, + ); + (1, message) + } + }; + eprintln!("{message}"); + + // Let the log reach its collector + Collector::shutdown(); + tokio::time::sleep(Duration::from_secs(1)).await; + code +} diff --git a/crates/migration/src/layout.rs b/crates/migration/src/layout.rs index f64c45e2..4ba97626 100644 --- a/crates/migration/src/layout.rs +++ b/crates/migration/src/layout.rs @@ -36,6 +36,33 @@ const LEASE_RENEWAL: Duration = Duration::from_secs(60); const WAIT_INTERVAL: Duration = Duration::from_secs(10); const WATCH_INTERVAL: Duration = Duration::from_secs(60); const UPDATING_DOCS: &str = "https://docs.inbuxa.org/install/updating/"; +/// K-S3: how long `--upgrade-store` waits for other nodes to show as +/// stopped before a step that needs them stopped. A node shows as stopped a +/// few minutes after its last heartbeat, so a pod that was just scaled down +/// still counts for a while. +const UPGRADE_ONLY_NODE_WAIT: Duration = Duration::from_secs(6 * 60); + +/// inbuxa: K-S3: who is running the steps. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Run { + /// A node starting: it waits for whatever is in the way, then serves. + Boot, + /// `--upgrade-store`: it upgrades and exits, and refuses rather than + /// waits for another node's upgrade. + UpgradeOnly, +} + +/// What a run of the steps came to. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Outcome { + /// The store was at this build's layout already, or another node + /// brought it there while this one waited. + Current, + /// This node upgraded it, from this version. + Upgraded { from: u32 }, + /// `--upgrade-store` refused, for this reason. + Refused(String), +} /// One step, from the version it starts at to the next. struct Step { @@ -103,27 +130,39 @@ pub async fn upgrade_layout( server: &Server, store_version: Option, new_install: bool, -) -> trc::Result<()> { + run: Run, +) -> trc::Result { let version = store_version.unwrap_or(if new_install { INBUXA_STORE_VERSION } else { 1 }); - if version < INBUXA_STORE_VERSION { - run_steps(server, server.registry().node_id()).await?; - } else if store_version.is_none() { - write_mark(server.store(), INBUXA_STORE_VERSION).await?; - } + let outcome = if version < INBUXA_STORE_VERSION { + run_steps_as(server, server.registry().node_id(), run).await? + } else { + if store_version.is_none() { + write_mark(server.store(), INBUXA_STORE_VERSION).await?; + } + Outcome::Current + }; // SL-7 - trc::event!( - Server(trc::ServerEvent::Startup), - Details = format!("Store layout version {INBUXA_STORE_VERSION}"), - ); + if !matches!(outcome, Outcome::Refused(_)) { + trc::event!( + Server(trc::ServerEvent::Startup), + Details = format!("Store layout version {INBUXA_STORE_VERSION}"), + ); + } - Ok(()) + Ok(outcome) } /// Runs every step from the store's version up to this build's, as node /// `node_id`, or waits for the node that runs them (SL-3). pub async fn run_steps(server: &Server, node_id: u16) -> trc::Result<()> { + run_steps_as(server, node_id, Run::Boot).await.map(|_| ()) +} + +/// As [`run_steps`]; under [`Run::UpgradeOnly`] another node's lease is a +/// refusal rather than something to wait out. +pub async fn run_steps_as(server: &Server, node_id: u16, run: Run) -> trc::Result { let store = server.store(); let mut waiting = false; @@ -137,15 +176,26 @@ pub async fn run_steps(server: &Server, node_id: u16) -> trc::Result<()> { Version = version, ); } - return Ok(()); + return Ok(Outcome::Current); } if let Some(lease) = Lease::take(store, node_id).await? { - let result = run_held(server, &lease, version).await; + let result = run_held(server, &lease, version, run).await; lease.release(store).await; return result; } + if run == Run::UpgradeOnly { + return Ok(Outcome::Refused(match Lease::holder(store).await? { + Some((holder, expires)) => format!( + "node {holder} holds the store layout lease and is upgrading the store; \ + it runs out in {}s unless that node renews it", + expires.saturating_sub(now()) + ), + None => "another node took the store layout lease first".to_string(), + })); + } + if !waiting { trc::event!( Server(trc::ServerEvent::Startup), @@ -158,10 +208,14 @@ pub async fn run_steps(server: &Server, node_id: u16) -> trc::Result<()> { } } -async fn run_held(server: &Server, lease: &Lease, version: u32) -> trc::Result<()> { +async fn run_held(server: &Server, lease: &Lease, version: u32, run: Run) -> trc::Result { let store = server.store(); // Another node may have run a step between the read and the lease let mut version = store_version(store).await?.unwrap_or(version); + let from = version; + if version >= INBUXA_STORE_VERSION { + return Ok(Outcome::Current); + } while version < INBUXA_STORE_VERSION { let Some(step) = STEPS.iter().find(|step| step.from == version) else { @@ -169,8 +223,14 @@ async fn run_held(server: &Server, lease: &Lease, version: u32) -> trc::Result<( "The data store is at layout version {version}, and this release has no step from it" )); }; - if !step.rolling_safe { - wait_for_other_nodes(server).await?; + if !step.rolling_safe + && let Some(running) = wait_for_other_nodes(server, run).await? + { + return Ok(Outcome::Refused(format!( + "the step from layout version {version} needs every other node stopped, \ + and these still show as running: {}", + running.join(", ") + ))); } // SL-8 @@ -206,16 +266,19 @@ async fn run_held(server: &Server, lease: &Lease, version: u32) -> trc::Result<( ); } - Ok(()) + Ok(Outcome::Upgraded { from }) } /// A step that isn't rolling-safe waits until no other node shows as active /// (SL-6). A single node never waits: only other node ids on other hosts -/// count, so this node's own earlier lease doesn't. -async fn wait_for_other_nodes(server: &Server) -> trc::Result<()> { +/// count, so this node's own earlier lease doesn't. Under +/// [`Run::UpgradeOnly`] it gives up after [`UPGRADE_ONLY_NODE_WAIT`] and +/// returns the nodes still running. +async fn wait_for_other_nodes(server: &Server, run: Run) -> trc::Result>> { let me = server.registry().node_id() as u64; let hostname = server.registry().local_hostname().to_string(); let mut logged: Option = None; + let started = Instant::now(); loop { let others = server @@ -232,7 +295,10 @@ async fn wait_for_other_nodes(server: &Server) -> trc::Result<()> { .map(|node| node.hostname) .collect::>(); if others.is_empty() { - return Ok(()); + return Ok(None); + } + if run == Run::UpgradeOnly && started.elapsed() >= UPGRADE_ONLY_NODE_WAIT { + return Ok(Some(others)); } if logged.is_none_or(|at| at.elapsed() >= Duration::from_secs(60)) { trc::event!( @@ -302,6 +368,19 @@ async fn swap_lease(store: &Store, current: Option<&[u8]>, new: Vec) -> trc: } impl Lease { + /// Who holds the lease and until when, if anyone does and it hasn't run out. + async fn holder(store: &Store) -> trc::Result> { + Ok(store + .get_value::(ValueKey::from(lease_class())) + .await + .caused_by(trc::location!())? + .and_then(|Raw(bytes)| { + let holder = bytes.as_slice().deserialize_be_u16(0).ok()?; + let expires = bytes.as_slice().deserialize_be_u64(2).ok()?; + (expires > now()).then_some((holder, expires)) + })) + } + /// Takes the lease if nobody holds it, it ran out, or it's this node's /// own (left by a run that stopped). async fn take(store: &Store, node_id: u16) -> trc::Result> { diff --git a/crates/migration/src/lib.rs b/crates/migration/src/lib.rs index f8b54a75..b5ea4405 100644 --- a/crates/migration/src/lib.rs +++ b/crates/migration/src/lib.rs @@ -23,6 +23,17 @@ pub mod layout; pub mod v016; pub async fn try_migrate(server: &Server) -> trc::Result<()> { + migrate(server, layout::Run::Boot).await.map(|_| ()) +} + +/// inbuxa: K-S3: `--upgrade-store`. What a start would do to the store +/// before serving, and nothing else; another node's upgrade is refused +/// rather than waited for. +pub async fn upgrade_store(server: &Server) -> trc::Result { + migrate(server, layout::Run::UpgradeOnly).await +} + +async fn migrate(server: &Server, run: layout::Run) -> trc::Result { // inbuxa: a store a newer release has changed is refused before anything // below writes to it. let store_version = layout::store_version(server.store()).await?; @@ -47,7 +58,7 @@ pub async fn try_migrate(server: &Server) -> trc::Result<()> { { Some(DATABASE_SCHEMA_VERSION) => { if !std::env::var("DANGER_FORCE_MIGRATE").is_ok_and(|v| v == "1") { - return layout::upgrade_layout(server, store_version, false).await; + return layout::upgrade_layout(server, store_version, false, run).await; } } Some(0..=4) => { @@ -74,7 +85,7 @@ pub async fn try_migrate(server: &Server) -> trc::Result<()> { _ => { if is_new_install(server).await.caused_by(trc::location!())? { write_schema_version(server).await?; - return layout::upgrade_layout(server, store_version, true).await; + return layout::upgrade_layout(server, store_version, true, run).await; } else { abort(concat!( "You must first upgrade to version 0.15, please read ", @@ -86,7 +97,7 @@ pub async fn try_migrate(server: &Server) -> trc::Result<()> { migrate_v0_16(server).await?; write_schema_version(server).await?; - layout::upgrade_layout(server, store_version, false).await + layout::upgrade_layout(server, store_version, false, run).await } async fn write_schema_version(server: &Server) -> trc::Result<()> { diff --git a/tests/src/store_version.rs b/tests/src/store_version.rs index c429e13f..12b41f64 100644 --- a/tests/src/store_version.rs +++ b/tests/src/store_version.rs @@ -94,6 +94,7 @@ async fn store_version() { layout_step_1_2(server).await; layout_lease(server).await; + upgrade_store_only(server).await; layout_step_2_3(server).await; } @@ -245,15 +246,14 @@ async fn write_lease(server: &Server, node_id: u16, expires_at: u64) { } async fn lease_held(server: &Server) -> bool { - server - .store() - .get_value::(AnyKey { - subspace: SUBSPACE_INBUXA, - key: migration::layout::LAYOUT_LEASE_KEY.to_vec(), - }) - .await - .unwrap() - .is_some() + // A lease is a node id and an expiry, so it's read as it is + get_raw( + server.store(), + SUBSPACE_INBUXA, + migration::layout::LAYOUT_LEASE_KEY, + ) + .await + .is_some() } // SL-3: a node that can't take the lease waits for the mark; a lease that @@ -309,6 +309,57 @@ async fn layout_lease(server: &Server) { assert!(!lease_held(server).await, "lease not released"); } +// inbuxa: K-S3: `--upgrade-store` upgrades and stops; it changes nothing on +// a current store, and refuses, rather than waits, while another node holds +// the lease. +async fn upgrade_store_only(server: &Server) { + use migration::layout::Outcome; + + // Current: nothing to do, twice over + write_mark(server, Some(INBUXA_STORE_VERSION)).await; + for _ in 0..2 { + assert_eq!( + migration::upgrade_store(server).await.unwrap(), + Outcome::Current + ); + assert_eq!(read_mark(server).await, Some(INBUXA_STORE_VERSION)); + } + + // Another node holds the lease: refused at once, and its lease is left + write_mark(server, Some(1)).await; + write_lease(server, 900, now() + 600).await; + let outcome = tokio::time::timeout(Duration::from_secs(5), migration::upgrade_store(server)) + .await + .expect("waited on another node's lease instead of refusing") + .unwrap(); + match outcome { + Outcome::Refused(reason) => assert!(reason.contains("node 900"), "{reason}"), + other => panic!("expected a refusal, got {other:?}"), + } + assert_eq!( + read_mark(server).await, + Some(1), + "upgraded under another lease" + ); + assert!( + lease_held(server).await, + "the other node's lease was removed" + ); + + // Once it's gone: upgraded, lease released, and a second run is a no-op + write_lease(server, 900, now() - 1).await; + assert_eq!( + migration::upgrade_store(server).await.unwrap(), + Outcome::Upgraded { from: 1 } + ); + assert_eq!(read_mark(server).await, Some(INBUXA_STORE_VERSION)); + assert!(!lease_held(server).await, "lease not released"); + assert_eq!( + migration::upgrade_store(server).await.unwrap(), + Outcome::Current + ); +} + async fn applications(server: &Server) -> Vec<(String, bool)> { let mut apps = server .registry()