--upgrade-store: upgrade the store layout, then exit #448

Merged
jcoffey-dev merged 1 commits from feat/upgrade-store into main 2026-11-18 15:56:02 +00:00
5 changed files with 331 additions and 89 deletions

No files matched your search

+106 -57
View File
@@ -30,6 +30,9 @@ pub struct BootManager {
pub inner: Arc<Inner>,
pub servers: Listeners,
pub ipc_rxs: IpcReceivers,
/// inbuxa: K-S3: started with `--upgrade-store`: no listeners, no
/// services; main upgrades the store's layout and exits
pub upgrade_store: bool,
}
pub struct IpcReceivers {
@@ -52,6 +55,7 @@ Options:
-e, --export <PATH> Export all store data to a specific path
-i, --import <PATH> Import store data from a specific path
-o, --console Open the store console
-u, --upgrade-store Upgrade the store's layout, then exit
-h, --help Print help
-V, --version Print version
@@ -61,6 +65,12 @@ index, which belongs to one search backend. An import into an empty store
queues the index to be rebuilt when the server next starts. EXPORT_TYPES
limits an export to some of: data, registry, blob, changelog, queue, report,
telemetry, tasks.
--upgrade-store does what a start does to the store before it serves, and
exits: 0 when the store is at this release's layout (whether it upgraded it
or found it there), 1 when the upgrade failed, 2 when it refused because
another node holds the upgrade lease, or still runs when a step needs every
other node stopped. It opens no ports and starts no services.
"#
);
@@ -69,6 +79,8 @@ enum StoreOp {
Export(BackupParams),
Import(PathBuf),
Console,
// inbuxa: K-S3
UpgradeStore,
None,
}
@@ -77,67 +89,88 @@ impl BootManager {
let mut config_path = std::env::var("CONFIG_PATH").ok();
let mut import_export = StoreOp::None;
if config_path.is_none() {
let mut args = std::env::args().skip(1);
// inbuxa: K-S3: the arguments are read with CONFIG_PATH set too, so
// `--upgrade-store` works where the path comes from the environment;
// and an option that takes no value no longer swallows the next one,
// so `--upgrade-store --config <PATH>` works in that order
let mut args = std::env::args().skip(1);
let has_args = args.len() > 0;
while let Some(arg) = args.next().and_then(|arg| {
arg.strip_prefix("--")
.or_else(|| arg.strip_prefix('-'))
.map(|arg| arg.to_string())
}) {
let (key, inline) = match arg.split_once('=') {
Some((key, value)) => (key.to_string(), Some(value.trim().to_string())),
None => (arg, None),
};
let takes_value = matches!(
key.as_str(),
"config" | "c" | "export" | "e" | "import" | "i"
);
let value = if takes_value {
inline.or_else(|| args.next())
} else {
inline
};
while let Some(arg) = args.next().and_then(|arg| {
arg.strip_prefix("--")
.or_else(|| arg.strip_prefix('-'))
.map(|arg| arg.to_string())
}) {
let (key, value) = if let Some((key, value)) = arg.split_once('=') {
(key.to_string(), Some(value.trim().to_string()))
} else {
(arg, args.next())
};
match (key.as_str(), value) {
("help" | "h", _) => {
eprintln!("{HELP}");
std::process::exit(0);
}
("version" | "V", _) => {
println!("{}", types::brand_version!());
std::process::exit(0);
}
("config" | "c", Some(value)) => {
config_path = Some(value);
}
("export" | "e", Some(value)) => {
import_export = StoreOp::Export(BackupParams::new(value.into()));
}
("import" | "i", Some(value)) => {
import_export = StoreOp::Import(value.into());
}
("console" | "o", None) => {
import_export = StoreOp::Console;
}
(_, None) => {
failed(&format!("Unrecognized command '{key}', try '--help'."));
}
(_, Some(_)) => failed(&format!(
"Missing value for argument '{key}', try '--help'."
)),
}
}
if config_path.is_none() {
if import_export == StoreOp::None {
match (key.as_str(), value) {
("help" | "h", _) => {
eprintln!("{HELP}");
} else {
eprintln!("Missing '--config' argument for import/export.")
std::process::exit(0);
}
std::process::exit(0);
("version" | "V", _) => {
println!("{}", types::brand_version!());
std::process::exit(0);
}
("config" | "c", Some(value)) => {
config_path = Some(value);
}
("export" | "e", Some(value)) => {
import_export = StoreOp::Export(BackupParams::new(value.into()));
}
("import" | "i", Some(value)) => {
import_export = StoreOp::Import(value.into());
}
("console" | "o", None) => {
import_export = StoreOp::Console;
}
("upgrade-store" | "u", None) => {
import_export = StoreOp::UpgradeStore;
}
(_, None) if takes_value => failed(&format!(
"Missing value for argument '{key}', try '--help'."
)),
(_, None) => {
failed(&format!("Unrecognized command '{key}', try '--help'."));
}
(_, Some(_)) => failed(&format!("Argument '{key}' takes no value, try '--help'.")),
}
}
if config_path.is_none() {
if import_export == StoreOp::None {
eprintln!("{HELP}");
std::process::exit(if has_args { 1 } else { 0 });
}
// A job that asked for a store operation must not pass without one
failed("Missing '--config' argument (or CONFIG_PATH) for this command.");
}
// Initialize registry
let upgrade_store = import_export == StoreOp::UpgradeStore;
let registry = RegistryStore::init(
PathBuf::from(config_path.unwrap()),
import_export == StoreOp::None,
import_export == StoreOp::None || upgrade_store,
)
.await
.failed("⚠️ Startup failed");
if upgrade_store && (registry.is_bootstrap_mode() || registry.is_recovery_mode()) {
failed(
"--upgrade-store needs a configured server: there is no data store to \
upgrade in bootstrap mode, and recovery mode is for a person at the console.",
);
}
let mut bootstrap = Bootstrap::new(registry).await;
// Add safe defaults if missing
@@ -146,8 +179,14 @@ impl BootManager {
}
// Start listeners
let mut servers = Listeners::parse(&mut bootstrap).await;
servers.bind_and_drop_priv(&mut bootstrap);
// inbuxa: K-S3: not for --upgrade-store, which serves nothing
let mut servers = if upgrade_store {
Listeners::default()
} else {
let servers = Listeners::parse(&mut bootstrap).await;
servers.bind_and_drop_priv(&mut bootstrap);
servers
};
// Parse storage
let storage = Storage::parse(&mut bootstrap).await;
@@ -156,7 +195,7 @@ impl BootManager {
let telemetry = Telemetry::parse(&mut bootstrap, &storage).await;
match import_export {
StoreOp::None => {
StoreOp::None | StoreOp::UpgradeStore => {
// Parse components
let core: Box<Core> =
Box::new(Box::pin(Core::parse(&mut bootstrap, storage)).await);
@@ -167,7 +206,14 @@ impl BootManager {
telemetry.enable();
if bootstrap.registry.is_bootstrap_mode() {
if upgrade_store {
trc::event!(
Server(trc::ServerEvent::Startup),
Details = "Started with --upgrade-store: upgrading the store layout, then exiting",
Hostname = bootstrap.registry.local_hostname().to_string(),
Version = types::brand_version!(),
);
} else if bootstrap.registry.is_bootstrap_mode() {
trc::event!(
Server(trc::ServerEvent::BootstrapMode),
Hostname = bootstrap.registry.local_hostname().to_string(),
@@ -216,7 +262,7 @@ impl BootManager {
cache,
});
if !bootstrap.registry.is_recovery_mode() {
if !bootstrap.registry.is_recovery_mode() && !upgrade_store {
// Load spam model
if let Err(err) = inner.build_server().spam_model_reload().await {
trc::error!(
@@ -235,9 +281,11 @@ impl BootManager {
}
// Parse TCP acceptors
servers
.parse_tcp_acceptors(&mut bootstrap, inner.clone())
.await;
if !upgrade_store {
servers
.parse_tcp_acceptors(&mut bootstrap, inner.clone())
.await;
}
// inbuxa: a reload isn't refused over objects that failed here
inner.build_server().record_build_errors(&bootstrap.errors);
@@ -251,6 +299,7 @@ impl BootManager {
bootstrap,
servers,
ipc_rxs,
upgrade_store,
}
}
StoreOp::Export(path) => {
+52
View File
@@ -60,6 +60,11 @@ async fn main() -> std::io::Result<()> {
// Load config and apply macros
let mut init = Box::pin(BootManager::init()).await;
// inbuxa: K-S3: --upgrade-store upgrades the store's layout and exits
if init.upgrade_store {
std::process::exit(upgrade_store_and_exit(&init.inner.build_server()).await);
}
// Migrate database
if let Err(err) = migration::try_migrate(&init.inner.build_server()).await {
trc::event!(
@@ -186,3 +191,50 @@ fn spawn_listener(
),
}
}
/// inbuxa: K-S3: the exit code of `--upgrade-store`: 0 when the store is at
/// this build's layout (upgraded now or already), 1 when the upgrade failed,
/// 2 when it was refused because another node is in the way.
async fn upgrade_store_and_exit(server: &common::Server) -> i32 {
use migration::layout::Outcome;
let (code, message) = match migration::upgrade_store(server).await {
Ok(Outcome::Current) => (
0,
format!(
"The store layout is at version {}: nothing to upgrade.",
common::INBUXA_STORE_VERSION
),
),
Ok(Outcome::Upgraded { from }) => (
0,
format!(
"Upgraded the store layout from version {from} to {}.",
common::INBUXA_STORE_VERSION
),
),
Ok(Outcome::Refused(reason)) => {
trc::event!(
Server(trc::ServerEvent::StartupError),
Details = "Refused to upgrade the store layout",
Reason = reason.clone(),
);
(2, format!("Refused to upgrade the store layout: {reason}."))
}
Err(err) => {
let message = format!("Failed to upgrade the store layout: {err}");
trc::event!(
Server(trc::ServerEvent::StartupError),
Details = "Failed to upgrade the store layout",
Reason = err,
);
(1, message)
}
};
eprintln!("{message}");
// Let the log reach its collector
Collector::shutdown();
tokio::time::sleep(Duration::from_secs(1)).await;
code
}
+99 -20
View File
@@ -36,6 +36,33 @@ const LEASE_RENEWAL: Duration = Duration::from_secs(60);
const WAIT_INTERVAL: Duration = Duration::from_secs(10);
const WATCH_INTERVAL: Duration = Duration::from_secs(60);
const UPDATING_DOCS: &str = "https://docs.inbuxa.org/install/updating/";
/// K-S3: how long `--upgrade-store` waits for other nodes to show as
/// stopped before a step that needs them stopped. A node shows as stopped a
/// few minutes after its last heartbeat, so a pod that was just scaled down
/// still counts for a while.
const UPGRADE_ONLY_NODE_WAIT: Duration = Duration::from_secs(6 * 60);
/// inbuxa: K-S3: who is running the steps.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Run {
/// A node starting: it waits for whatever is in the way, then serves.
Boot,
/// `--upgrade-store`: it upgrades and exits, and refuses rather than
/// waits for another node's upgrade.
UpgradeOnly,
}
/// What a run of the steps came to.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Outcome {
/// The store was at this build's layout already, or another node
/// brought it there while this one waited.
Current,
/// This node upgraded it, from this version.
Upgraded { from: u32 },
/// `--upgrade-store` refused, for this reason.
Refused(String),
}
/// One step, from the version it starts at to the next.
struct Step {
@@ -103,27 +130,39 @@ pub async fn upgrade_layout(
server: &Server,
store_version: Option<u32>,
new_install: bool,
) -> trc::Result<()> {
run: Run,
) -> trc::Result<Outcome> {
let version = store_version.unwrap_or(if new_install { INBUXA_STORE_VERSION } else { 1 });
if version < INBUXA_STORE_VERSION {
run_steps(server, server.registry().node_id()).await?;
} else if store_version.is_none() {
write_mark(server.store(), INBUXA_STORE_VERSION).await?;
}
let outcome = if version < INBUXA_STORE_VERSION {
run_steps_as(server, server.registry().node_id(), run).await?
} else {
if store_version.is_none() {
write_mark(server.store(), INBUXA_STORE_VERSION).await?;
}
Outcome::Current
};
// SL-7
trc::event!(
Server(trc::ServerEvent::Startup),
Details = format!("Store layout version {INBUXA_STORE_VERSION}"),
);
if !matches!(outcome, Outcome::Refused(_)) {
trc::event!(
Server(trc::ServerEvent::Startup),
Details = format!("Store layout version {INBUXA_STORE_VERSION}"),
);
}
Ok(())
Ok(outcome)
}
/// Runs every step from the store's version up to this build's, as node
/// `node_id`, or waits for the node that runs them (SL-3).
pub async fn run_steps(server: &Server, node_id: u16) -> trc::Result<()> {
run_steps_as(server, node_id, Run::Boot).await.map(|_| ())
}
/// As [`run_steps`]; under [`Run::UpgradeOnly`] another node's lease is a
/// refusal rather than something to wait out.
pub async fn run_steps_as(server: &Server, node_id: u16, run: Run) -> trc::Result<Outcome> {
let store = server.store();
let mut waiting = false;
@@ -137,15 +176,26 @@ pub async fn run_steps(server: &Server, node_id: u16) -> trc::Result<()> {
Version = version,
);
}
return Ok(());
return Ok(Outcome::Current);
}
if let Some(lease) = Lease::take(store, node_id).await? {
let result = run_held(server, &lease, version).await;
let result = run_held(server, &lease, version, run).await;
lease.release(store).await;
return result;
}
if run == Run::UpgradeOnly {
return Ok(Outcome::Refused(match Lease::holder(store).await? {
Some((holder, expires)) => format!(
"node {holder} holds the store layout lease and is upgrading the store; \
it runs out in {}s unless that node renews it",
expires.saturating_sub(now())
),
None => "another node took the store layout lease first".to_string(),
}));
}
if !waiting {
trc::event!(
Server(trc::ServerEvent::Startup),
@@ -158,10 +208,14 @@ pub async fn run_steps(server: &Server, node_id: u16) -> trc::Result<()> {
}
}
async fn run_held(server: &Server, lease: &Lease, version: u32) -> trc::Result<()> {
async fn run_held(server: &Server, lease: &Lease, version: u32, run: Run) -> trc::Result<Outcome> {
let store = server.store();
// Another node may have run a step between the read and the lease
let mut version = store_version(store).await?.unwrap_or(version);
let from = version;
if version >= INBUXA_STORE_VERSION {
return Ok(Outcome::Current);
}
while version < INBUXA_STORE_VERSION {
let Some(step) = STEPS.iter().find(|step| step.from == version) else {
@@ -169,8 +223,14 @@ async fn run_held(server: &Server, lease: &Lease, version: u32) -> trc::Result<(
"The data store is at layout version {version}, and this release has no step from it"
));
};
if !step.rolling_safe {
wait_for_other_nodes(server).await?;
if !step.rolling_safe
&& let Some(running) = wait_for_other_nodes(server, run).await?
{
return Ok(Outcome::Refused(format!(
"the step from layout version {version} needs every other node stopped, \
and these still show as running: {}",
running.join(", ")
)));
}
// SL-8
@@ -206,16 +266,19 @@ async fn run_held(server: &Server, lease: &Lease, version: u32) -> trc::Result<(
);
}
Ok(())
Ok(Outcome::Upgraded { from })
}
/// A step that isn't rolling-safe waits until no other node shows as active
/// (SL-6). A single node never waits: only other node ids on other hosts
/// count, so this node's own earlier lease doesn't.
async fn wait_for_other_nodes(server: &Server) -> trc::Result<()> {
/// count, so this node's own earlier lease doesn't. Under
/// [`Run::UpgradeOnly`] it gives up after [`UPGRADE_ONLY_NODE_WAIT`] and
/// returns the nodes still running.
async fn wait_for_other_nodes(server: &Server, run: Run) -> trc::Result<Option<Vec<String>>> {
let me = server.registry().node_id() as u64;
let hostname = server.registry().local_hostname().to_string();
let mut logged: Option<Instant> = None;
let started = Instant::now();
loop {
let others = server
@@ -232,7 +295,10 @@ async fn wait_for_other_nodes(server: &Server) -> trc::Result<()> {
.map(|node| node.hostname)
.collect::<Vec<_>>();
if others.is_empty() {
return Ok(());
return Ok(None);
}
if run == Run::UpgradeOnly && started.elapsed() >= UPGRADE_ONLY_NODE_WAIT {
return Ok(Some(others));
}
if logged.is_none_or(|at| at.elapsed() >= Duration::from_secs(60)) {
trc::event!(
@@ -302,6 +368,19 @@ async fn swap_lease(store: &Store, current: Option<&[u8]>, new: Vec<u8>) -> trc:
}
impl Lease {
/// Who holds the lease and until when, if anyone does and it hasn't run out.
async fn holder(store: &Store) -> trc::Result<Option<(u16, u64)>> {
Ok(store
.get_value::<Raw>(ValueKey::from(lease_class()))
.await
.caused_by(trc::location!())?
.and_then(|Raw(bytes)| {
let holder = bytes.as_slice().deserialize_be_u16(0).ok()?;
let expires = bytes.as_slice().deserialize_be_u64(2).ok()?;
(expires > now()).then_some((holder, expires))
}))
}
/// Takes the lease if nobody holds it, it ran out, or it's this node's
/// own (left by a run that stopped).
async fn take(store: &Store, node_id: u16) -> trc::Result<Option<Lease>> {
+14 -3
View File
@@ -23,6 +23,17 @@ pub mod layout;
pub mod v016;
pub async fn try_migrate(server: &Server) -> trc::Result<()> {
migrate(server, layout::Run::Boot).await.map(|_| ())
}
/// inbuxa: K-S3: `--upgrade-store`. What a start would do to the store
/// before serving, and nothing else; another node's upgrade is refused
/// rather than waited for.
pub async fn upgrade_store(server: &Server) -> trc::Result<layout::Outcome> {
migrate(server, layout::Run::UpgradeOnly).await
}
async fn migrate(server: &Server, run: layout::Run) -> trc::Result<layout::Outcome> {
// inbuxa: a store a newer release has changed is refused before anything
// below writes to it.
let store_version = layout::store_version(server.store()).await?;
@@ -47,7 +58,7 @@ pub async fn try_migrate(server: &Server) -> trc::Result<()> {
{
Some(DATABASE_SCHEMA_VERSION) => {
if !std::env::var("DANGER_FORCE_MIGRATE").is_ok_and(|v| v == "1") {
return layout::upgrade_layout(server, store_version, false).await;
return layout::upgrade_layout(server, store_version, false, run).await;
}
}
Some(0..=4) => {
@@ -74,7 +85,7 @@ pub async fn try_migrate(server: &Server) -> trc::Result<()> {
_ => {
if is_new_install(server).await.caused_by(trc::location!())? {
write_schema_version(server).await?;
return layout::upgrade_layout(server, store_version, true).await;
return layout::upgrade_layout(server, store_version, true, run).await;
} else {
abort(concat!(
"You must first upgrade to version 0.15, please read ",
@@ -86,7 +97,7 @@ pub async fn try_migrate(server: &Server) -> trc::Result<()> {
migrate_v0_16(server).await?;
write_schema_version(server).await?;
layout::upgrade_layout(server, store_version, false).await
layout::upgrade_layout(server, store_version, false, run).await
}
async fn write_schema_version(server: &Server) -> trc::Result<()> {
+60 -9
View File
@@ -94,6 +94,7 @@ async fn store_version() {
layout_step_1_2(server).await;
layout_lease(server).await;
upgrade_store_only(server).await;
layout_step_2_3(server).await;
}
@@ -245,15 +246,14 @@ async fn write_lease(server: &Server, node_id: u16, expires_at: u64) {
}
async fn lease_held(server: &Server) -> bool {
server
.store()
.get_value::<u64>(AnyKey {
subspace: SUBSPACE_INBUXA,
key: migration::layout::LAYOUT_LEASE_KEY.to_vec(),
})
.await
.unwrap()
.is_some()
// A lease is a node id and an expiry, so it's read as it is
get_raw(
server.store(),
SUBSPACE_INBUXA,
migration::layout::LAYOUT_LEASE_KEY,
)
.await
.is_some()
}
// SL-3: a node that can't take the lease waits for the mark; a lease that
@@ -309,6 +309,57 @@ async fn layout_lease(server: &Server) {
assert!(!lease_held(server).await, "lease not released");
}
// inbuxa: K-S3: `--upgrade-store` upgrades and stops; it changes nothing on
// a current store, and refuses, rather than waits, while another node holds
// the lease.
async fn upgrade_store_only(server: &Server) {
use migration::layout::Outcome;
// Current: nothing to do, twice over
write_mark(server, Some(INBUXA_STORE_VERSION)).await;
for _ in 0..2 {
assert_eq!(
migration::upgrade_store(server).await.unwrap(),
Outcome::Current
);
assert_eq!(read_mark(server).await, Some(INBUXA_STORE_VERSION));
}
// Another node holds the lease: refused at once, and its lease is left
write_mark(server, Some(1)).await;
write_lease(server, 900, now() + 600).await;
let outcome = tokio::time::timeout(Duration::from_secs(5), migration::upgrade_store(server))
.await
.expect("waited on another node's lease instead of refusing")
.unwrap();
match outcome {
Outcome::Refused(reason) => assert!(reason.contains("node 900"), "{reason}"),
other => panic!("expected a refusal, got {other:?}"),
}
assert_eq!(
read_mark(server).await,
Some(1),
"upgraded under another lease"
);
assert!(
lease_held(server).await,
"the other node's lease was removed"
);
// Once it's gone: upgraded, lease released, and a second run is a no-op
write_lease(server, 900, now() - 1).await;
assert_eq!(
migration::upgrade_store(server).await.unwrap(),
Outcome::Upgraded { from: 1 }
);
assert_eq!(read_mark(server).await, Some(INBUXA_STORE_VERSION));
assert!(!lease_held(server).await, "lease not released");
assert_eq!(
migration::upgrade_store(server).await.unwrap(),
Outcome::Current
);
}
async fn applications(server: &Server) -> Vec<(String, bool)> {
let mut apps = server
.registry()