--upgrade-store: upgrade the store layout, then exit #448
No files matched your search
@@ -30,6 +30,9 @@ pub struct BootManager {
|
||||
pub inner: Arc<Inner>,
|
||||
pub servers: Listeners,
|
||||
pub ipc_rxs: IpcReceivers,
|
||||
/// inbuxa: K-S3: started with `--upgrade-store`: no listeners, no
|
||||
/// services; main upgrades the store's layout and exits
|
||||
pub upgrade_store: bool,
|
||||
}
|
||||
|
||||
pub struct IpcReceivers {
|
||||
@@ -52,6 +55,7 @@ Options:
|
||||
-e, --export <PATH> Export all store data to a specific path
|
||||
-i, --import <PATH> Import store data from a specific path
|
||||
-o, --console Open the store console
|
||||
-u, --upgrade-store Upgrade the store's layout, then exit
|
||||
-h, --help Print help
|
||||
-V, --version Print version
|
||||
|
||||
@@ -61,6 +65,12 @@ index, which belongs to one search backend. An import into an empty store
|
||||
queues the index to be rebuilt when the server next starts. EXPORT_TYPES
|
||||
limits an export to some of: data, registry, blob, changelog, queue, report,
|
||||
telemetry, tasks.
|
||||
|
||||
--upgrade-store does what a start does to the store before it serves, and
|
||||
exits: 0 when the store is at this release's layout (whether it upgraded it
|
||||
or found it there), 1 when the upgrade failed, 2 when it refused because
|
||||
another node holds the upgrade lease, or still runs when a step needs every
|
||||
other node stopped. It opens no ports and starts no services.
|
||||
"#
|
||||
);
|
||||
|
||||
@@ -69,6 +79,8 @@ enum StoreOp {
|
||||
Export(BackupParams),
|
||||
Import(PathBuf),
|
||||
Console,
|
||||
// inbuxa: K-S3
|
||||
UpgradeStore,
|
||||
None,
|
||||
}
|
||||
|
||||
@@ -77,67 +89,88 @@ impl BootManager {
|
||||
let mut config_path = std::env::var("CONFIG_PATH").ok();
|
||||
let mut import_export = StoreOp::None;
|
||||
|
||||
if config_path.is_none() {
|
||||
let mut args = std::env::args().skip(1);
|
||||
// inbuxa: K-S3: the arguments are read with CONFIG_PATH set too, so
|
||||
// `--upgrade-store` works where the path comes from the environment;
|
||||
// and an option that takes no value no longer swallows the next one,
|
||||
// so `--upgrade-store --config <PATH>` works in that order
|
||||
let mut args = std::env::args().skip(1);
|
||||
let has_args = args.len() > 0;
|
||||
while let Some(arg) = args.next().and_then(|arg| {
|
||||
arg.strip_prefix("--")
|
||||
.or_else(|| arg.strip_prefix('-'))
|
||||
.map(|arg| arg.to_string())
|
||||
}) {
|
||||
let (key, inline) = match arg.split_once('=') {
|
||||
Some((key, value)) => (key.to_string(), Some(value.trim().to_string())),
|
||||
None => (arg, None),
|
||||
};
|
||||
let takes_value = matches!(
|
||||
key.as_str(),
|
||||
"config" | "c" | "export" | "e" | "import" | "i"
|
||||
);
|
||||
let value = if takes_value {
|
||||
inline.or_else(|| args.next())
|
||||
} else {
|
||||
inline
|
||||
};
|
||||
|
||||
while let Some(arg) = args.next().and_then(|arg| {
|
||||
arg.strip_prefix("--")
|
||||
.or_else(|| arg.strip_prefix('-'))
|
||||
.map(|arg| arg.to_string())
|
||||
}) {
|
||||
let (key, value) = if let Some((key, value)) = arg.split_once('=') {
|
||||
(key.to_string(), Some(value.trim().to_string()))
|
||||
} else {
|
||||
(arg, args.next())
|
||||
};
|
||||
|
||||
match (key.as_str(), value) {
|
||||
("help" | "h", _) => {
|
||||
eprintln!("{HELP}");
|
||||
std::process::exit(0);
|
||||
}
|
||||
("version" | "V", _) => {
|
||||
println!("{}", types::brand_version!());
|
||||
std::process::exit(0);
|
||||
}
|
||||
("config" | "c", Some(value)) => {
|
||||
config_path = Some(value);
|
||||
}
|
||||
("export" | "e", Some(value)) => {
|
||||
import_export = StoreOp::Export(BackupParams::new(value.into()));
|
||||
}
|
||||
("import" | "i", Some(value)) => {
|
||||
import_export = StoreOp::Import(value.into());
|
||||
}
|
||||
("console" | "o", None) => {
|
||||
import_export = StoreOp::Console;
|
||||
}
|
||||
(_, None) => {
|
||||
failed(&format!("Unrecognized command '{key}', try '--help'."));
|
||||
}
|
||||
(_, Some(_)) => failed(&format!(
|
||||
"Missing value for argument '{key}', try '--help'."
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
if config_path.is_none() {
|
||||
if import_export == StoreOp::None {
|
||||
match (key.as_str(), value) {
|
||||
("help" | "h", _) => {
|
||||
eprintln!("{HELP}");
|
||||
} else {
|
||||
eprintln!("Missing '--config' argument for import/export.")
|
||||
std::process::exit(0);
|
||||
}
|
||||
std::process::exit(0);
|
||||
("version" | "V", _) => {
|
||||
println!("{}", types::brand_version!());
|
||||
std::process::exit(0);
|
||||
}
|
||||
("config" | "c", Some(value)) => {
|
||||
config_path = Some(value);
|
||||
}
|
||||
("export" | "e", Some(value)) => {
|
||||
import_export = StoreOp::Export(BackupParams::new(value.into()));
|
||||
}
|
||||
("import" | "i", Some(value)) => {
|
||||
import_export = StoreOp::Import(value.into());
|
||||
}
|
||||
("console" | "o", None) => {
|
||||
import_export = StoreOp::Console;
|
||||
}
|
||||
("upgrade-store" | "u", None) => {
|
||||
import_export = StoreOp::UpgradeStore;
|
||||
}
|
||||
(_, None) if takes_value => failed(&format!(
|
||||
"Missing value for argument '{key}', try '--help'."
|
||||
)),
|
||||
(_, None) => {
|
||||
failed(&format!("Unrecognized command '{key}', try '--help'."));
|
||||
}
|
||||
(_, Some(_)) => failed(&format!("Argument '{key}' takes no value, try '--help'.")),
|
||||
}
|
||||
}
|
||||
|
||||
if config_path.is_none() {
|
||||
if import_export == StoreOp::None {
|
||||
eprintln!("{HELP}");
|
||||
std::process::exit(if has_args { 1 } else { 0 });
|
||||
}
|
||||
// A job that asked for a store operation must not pass without one
|
||||
failed("Missing '--config' argument (or CONFIG_PATH) for this command.");
|
||||
}
|
||||
|
||||
// Initialize registry
|
||||
let upgrade_store = import_export == StoreOp::UpgradeStore;
|
||||
let registry = RegistryStore::init(
|
||||
PathBuf::from(config_path.unwrap()),
|
||||
import_export == StoreOp::None,
|
||||
import_export == StoreOp::None || upgrade_store,
|
||||
)
|
||||
.await
|
||||
.failed("⚠️ Startup failed");
|
||||
if upgrade_store && (registry.is_bootstrap_mode() || registry.is_recovery_mode()) {
|
||||
failed(
|
||||
"--upgrade-store needs a configured server: there is no data store to \
|
||||
upgrade in bootstrap mode, and recovery mode is for a person at the console.",
|
||||
);
|
||||
}
|
||||
let mut bootstrap = Bootstrap::new(registry).await;
|
||||
|
||||
// Add safe defaults if missing
|
||||
@@ -146,8 +179,14 @@ impl BootManager {
|
||||
}
|
||||
|
||||
// Start listeners
|
||||
let mut servers = Listeners::parse(&mut bootstrap).await;
|
||||
servers.bind_and_drop_priv(&mut bootstrap);
|
||||
// inbuxa: K-S3: not for --upgrade-store, which serves nothing
|
||||
let mut servers = if upgrade_store {
|
||||
Listeners::default()
|
||||
} else {
|
||||
let servers = Listeners::parse(&mut bootstrap).await;
|
||||
servers.bind_and_drop_priv(&mut bootstrap);
|
||||
servers
|
||||
};
|
||||
|
||||
// Parse storage
|
||||
let storage = Storage::parse(&mut bootstrap).await;
|
||||
@@ -156,7 +195,7 @@ impl BootManager {
|
||||
let telemetry = Telemetry::parse(&mut bootstrap, &storage).await;
|
||||
|
||||
match import_export {
|
||||
StoreOp::None => {
|
||||
StoreOp::None | StoreOp::UpgradeStore => {
|
||||
// Parse components
|
||||
let core: Box<Core> =
|
||||
Box::new(Box::pin(Core::parse(&mut bootstrap, storage)).await);
|
||||
@@ -167,7 +206,14 @@ impl BootManager {
|
||||
|
||||
telemetry.enable();
|
||||
|
||||
if bootstrap.registry.is_bootstrap_mode() {
|
||||
if upgrade_store {
|
||||
trc::event!(
|
||||
Server(trc::ServerEvent::Startup),
|
||||
Details = "Started with --upgrade-store: upgrading the store layout, then exiting",
|
||||
Hostname = bootstrap.registry.local_hostname().to_string(),
|
||||
Version = types::brand_version!(),
|
||||
);
|
||||
} else if bootstrap.registry.is_bootstrap_mode() {
|
||||
trc::event!(
|
||||
Server(trc::ServerEvent::BootstrapMode),
|
||||
Hostname = bootstrap.registry.local_hostname().to_string(),
|
||||
@@ -216,7 +262,7 @@ impl BootManager {
|
||||
cache,
|
||||
});
|
||||
|
||||
if !bootstrap.registry.is_recovery_mode() {
|
||||
if !bootstrap.registry.is_recovery_mode() && !upgrade_store {
|
||||
// Load spam model
|
||||
if let Err(err) = inner.build_server().spam_model_reload().await {
|
||||
trc::error!(
|
||||
@@ -235,9 +281,11 @@ impl BootManager {
|
||||
}
|
||||
|
||||
// Parse TCP acceptors
|
||||
servers
|
||||
.parse_tcp_acceptors(&mut bootstrap, inner.clone())
|
||||
.await;
|
||||
if !upgrade_store {
|
||||
servers
|
||||
.parse_tcp_acceptors(&mut bootstrap, inner.clone())
|
||||
.await;
|
||||
}
|
||||
|
||||
// inbuxa: a reload isn't refused over objects that failed here
|
||||
inner.build_server().record_build_errors(&bootstrap.errors);
|
||||
@@ -251,6 +299,7 @@ impl BootManager {
|
||||
bootstrap,
|
||||
servers,
|
||||
ipc_rxs,
|
||||
upgrade_store,
|
||||
}
|
||||
}
|
||||
StoreOp::Export(path) => {
|
||||
|
||||
@@ -60,6 +60,11 @@ async fn main() -> std::io::Result<()> {
|
||||
// Load config and apply macros
|
||||
let mut init = Box::pin(BootManager::init()).await;
|
||||
|
||||
// inbuxa: K-S3: --upgrade-store upgrades the store's layout and exits
|
||||
if init.upgrade_store {
|
||||
std::process::exit(upgrade_store_and_exit(&init.inner.build_server()).await);
|
||||
}
|
||||
|
||||
// Migrate database
|
||||
if let Err(err) = migration::try_migrate(&init.inner.build_server()).await {
|
||||
trc::event!(
|
||||
@@ -186,3 +191,50 @@ fn spawn_listener(
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// inbuxa: K-S3: the exit code of `--upgrade-store`: 0 when the store is at
|
||||
/// this build's layout (upgraded now or already), 1 when the upgrade failed,
|
||||
/// 2 when it was refused because another node is in the way.
|
||||
async fn upgrade_store_and_exit(server: &common::Server) -> i32 {
|
||||
use migration::layout::Outcome;
|
||||
|
||||
let (code, message) = match migration::upgrade_store(server).await {
|
||||
Ok(Outcome::Current) => (
|
||||
0,
|
||||
format!(
|
||||
"The store layout is at version {}: nothing to upgrade.",
|
||||
common::INBUXA_STORE_VERSION
|
||||
),
|
||||
),
|
||||
Ok(Outcome::Upgraded { from }) => (
|
||||
0,
|
||||
format!(
|
||||
"Upgraded the store layout from version {from} to {}.",
|
||||
common::INBUXA_STORE_VERSION
|
||||
),
|
||||
),
|
||||
Ok(Outcome::Refused(reason)) => {
|
||||
trc::event!(
|
||||
Server(trc::ServerEvent::StartupError),
|
||||
Details = "Refused to upgrade the store layout",
|
||||
Reason = reason.clone(),
|
||||
);
|
||||
(2, format!("Refused to upgrade the store layout: {reason}."))
|
||||
}
|
||||
Err(err) => {
|
||||
let message = format!("Failed to upgrade the store layout: {err}");
|
||||
trc::event!(
|
||||
Server(trc::ServerEvent::StartupError),
|
||||
Details = "Failed to upgrade the store layout",
|
||||
Reason = err,
|
||||
);
|
||||
(1, message)
|
||||
}
|
||||
};
|
||||
eprintln!("{message}");
|
||||
|
||||
// Let the log reach its collector
|
||||
Collector::shutdown();
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
code
|
||||
}
|
||||
@@ -36,6 +36,33 @@ const LEASE_RENEWAL: Duration = Duration::from_secs(60);
|
||||
const WAIT_INTERVAL: Duration = Duration::from_secs(10);
|
||||
const WATCH_INTERVAL: Duration = Duration::from_secs(60);
|
||||
const UPDATING_DOCS: &str = "https://docs.inbuxa.org/install/updating/";
|
||||
/// K-S3: how long `--upgrade-store` waits for other nodes to show as
|
||||
/// stopped before a step that needs them stopped. A node shows as stopped a
|
||||
/// few minutes after its last heartbeat, so a pod that was just scaled down
|
||||
/// still counts for a while.
|
||||
const UPGRADE_ONLY_NODE_WAIT: Duration = Duration::from_secs(6 * 60);
|
||||
|
||||
/// inbuxa: K-S3: who is running the steps.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Run {
|
||||
/// A node starting: it waits for whatever is in the way, then serves.
|
||||
Boot,
|
||||
/// `--upgrade-store`: it upgrades and exits, and refuses rather than
|
||||
/// waits for another node's upgrade.
|
||||
UpgradeOnly,
|
||||
}
|
||||
|
||||
/// What a run of the steps came to.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Outcome {
|
||||
/// The store was at this build's layout already, or another node
|
||||
/// brought it there while this one waited.
|
||||
Current,
|
||||
/// This node upgraded it, from this version.
|
||||
Upgraded { from: u32 },
|
||||
/// `--upgrade-store` refused, for this reason.
|
||||
Refused(String),
|
||||
}
|
||||
|
||||
/// One step, from the version it starts at to the next.
|
||||
struct Step {
|
||||
@@ -103,27 +130,39 @@ pub async fn upgrade_layout(
|
||||
server: &Server,
|
||||
store_version: Option<u32>,
|
||||
new_install: bool,
|
||||
) -> trc::Result<()> {
|
||||
run: Run,
|
||||
) -> trc::Result<Outcome> {
|
||||
let version = store_version.unwrap_or(if new_install { INBUXA_STORE_VERSION } else { 1 });
|
||||
|
||||
if version < INBUXA_STORE_VERSION {
|
||||
run_steps(server, server.registry().node_id()).await?;
|
||||
} else if store_version.is_none() {
|
||||
write_mark(server.store(), INBUXA_STORE_VERSION).await?;
|
||||
}
|
||||
let outcome = if version < INBUXA_STORE_VERSION {
|
||||
run_steps_as(server, server.registry().node_id(), run).await?
|
||||
} else {
|
||||
if store_version.is_none() {
|
||||
write_mark(server.store(), INBUXA_STORE_VERSION).await?;
|
||||
}
|
||||
Outcome::Current
|
||||
};
|
||||
|
||||
// SL-7
|
||||
trc::event!(
|
||||
Server(trc::ServerEvent::Startup),
|
||||
Details = format!("Store layout version {INBUXA_STORE_VERSION}"),
|
||||
);
|
||||
if !matches!(outcome, Outcome::Refused(_)) {
|
||||
trc::event!(
|
||||
Server(trc::ServerEvent::Startup),
|
||||
Details = format!("Store layout version {INBUXA_STORE_VERSION}"),
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
/// Runs every step from the store's version up to this build's, as node
|
||||
/// `node_id`, or waits for the node that runs them (SL-3).
|
||||
pub async fn run_steps(server: &Server, node_id: u16) -> trc::Result<()> {
|
||||
run_steps_as(server, node_id, Run::Boot).await.map(|_| ())
|
||||
}
|
||||
|
||||
/// As [`run_steps`]; under [`Run::UpgradeOnly`] another node's lease is a
|
||||
/// refusal rather than something to wait out.
|
||||
pub async fn run_steps_as(server: &Server, node_id: u16, run: Run) -> trc::Result<Outcome> {
|
||||
let store = server.store();
|
||||
let mut waiting = false;
|
||||
|
||||
@@ -137,15 +176,26 @@ pub async fn run_steps(server: &Server, node_id: u16) -> trc::Result<()> {
|
||||
Version = version,
|
||||
);
|
||||
}
|
||||
return Ok(());
|
||||
return Ok(Outcome::Current);
|
||||
}
|
||||
|
||||
if let Some(lease) = Lease::take(store, node_id).await? {
|
||||
let result = run_held(server, &lease, version).await;
|
||||
let result = run_held(server, &lease, version, run).await;
|
||||
lease.release(store).await;
|
||||
return result;
|
||||
}
|
||||
|
||||
if run == Run::UpgradeOnly {
|
||||
return Ok(Outcome::Refused(match Lease::holder(store).await? {
|
||||
Some((holder, expires)) => format!(
|
||||
"node {holder} holds the store layout lease and is upgrading the store; \
|
||||
it runs out in {}s unless that node renews it",
|
||||
expires.saturating_sub(now())
|
||||
),
|
||||
None => "another node took the store layout lease first".to_string(),
|
||||
}));
|
||||
}
|
||||
|
||||
if !waiting {
|
||||
trc::event!(
|
||||
Server(trc::ServerEvent::Startup),
|
||||
@@ -158,10 +208,14 @@ pub async fn run_steps(server: &Server, node_id: u16) -> trc::Result<()> {
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_held(server: &Server, lease: &Lease, version: u32) -> trc::Result<()> {
|
||||
async fn run_held(server: &Server, lease: &Lease, version: u32, run: Run) -> trc::Result<Outcome> {
|
||||
let store = server.store();
|
||||
// Another node may have run a step between the read and the lease
|
||||
let mut version = store_version(store).await?.unwrap_or(version);
|
||||
let from = version;
|
||||
if version >= INBUXA_STORE_VERSION {
|
||||
return Ok(Outcome::Current);
|
||||
}
|
||||
|
||||
while version < INBUXA_STORE_VERSION {
|
||||
let Some(step) = STEPS.iter().find(|step| step.from == version) else {
|
||||
@@ -169,8 +223,14 @@ async fn run_held(server: &Server, lease: &Lease, version: u32) -> trc::Result<(
|
||||
"The data store is at layout version {version}, and this release has no step from it"
|
||||
));
|
||||
};
|
||||
if !step.rolling_safe {
|
||||
wait_for_other_nodes(server).await?;
|
||||
if !step.rolling_safe
|
||||
&& let Some(running) = wait_for_other_nodes(server, run).await?
|
||||
{
|
||||
return Ok(Outcome::Refused(format!(
|
||||
"the step from layout version {version} needs every other node stopped, \
|
||||
and these still show as running: {}",
|
||||
running.join(", ")
|
||||
)));
|
||||
}
|
||||
|
||||
// SL-8
|
||||
@@ -206,16 +266,19 @@ async fn run_held(server: &Server, lease: &Lease, version: u32) -> trc::Result<(
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
Ok(Outcome::Upgraded { from })
|
||||
}
|
||||
|
||||
/// A step that isn't rolling-safe waits until no other node shows as active
|
||||
/// (SL-6). A single node never waits: only other node ids on other hosts
|
||||
/// count, so this node's own earlier lease doesn't.
|
||||
async fn wait_for_other_nodes(server: &Server) -> trc::Result<()> {
|
||||
/// count, so this node's own earlier lease doesn't. Under
|
||||
/// [`Run::UpgradeOnly`] it gives up after [`UPGRADE_ONLY_NODE_WAIT`] and
|
||||
/// returns the nodes still running.
|
||||
async fn wait_for_other_nodes(server: &Server, run: Run) -> trc::Result<Option<Vec<String>>> {
|
||||
let me = server.registry().node_id() as u64;
|
||||
let hostname = server.registry().local_hostname().to_string();
|
||||
let mut logged: Option<Instant> = None;
|
||||
let started = Instant::now();
|
||||
|
||||
loop {
|
||||
let others = server
|
||||
@@ -232,7 +295,10 @@ async fn wait_for_other_nodes(server: &Server) -> trc::Result<()> {
|
||||
.map(|node| node.hostname)
|
||||
.collect::<Vec<_>>();
|
||||
if others.is_empty() {
|
||||
return Ok(());
|
||||
return Ok(None);
|
||||
}
|
||||
if run == Run::UpgradeOnly && started.elapsed() >= UPGRADE_ONLY_NODE_WAIT {
|
||||
return Ok(Some(others));
|
||||
}
|
||||
if logged.is_none_or(|at| at.elapsed() >= Duration::from_secs(60)) {
|
||||
trc::event!(
|
||||
@@ -302,6 +368,19 @@ async fn swap_lease(store: &Store, current: Option<&[u8]>, new: Vec<u8>) -> trc:
|
||||
}
|
||||
|
||||
impl Lease {
|
||||
/// Who holds the lease and until when, if anyone does and it hasn't run out.
|
||||
async fn holder(store: &Store) -> trc::Result<Option<(u16, u64)>> {
|
||||
Ok(store
|
||||
.get_value::<Raw>(ValueKey::from(lease_class()))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.and_then(|Raw(bytes)| {
|
||||
let holder = bytes.as_slice().deserialize_be_u16(0).ok()?;
|
||||
let expires = bytes.as_slice().deserialize_be_u64(2).ok()?;
|
||||
(expires > now()).then_some((holder, expires))
|
||||
}))
|
||||
}
|
||||
|
||||
/// Takes the lease if nobody holds it, it ran out, or it's this node's
|
||||
/// own (left by a run that stopped).
|
||||
async fn take(store: &Store, node_id: u16) -> trc::Result<Option<Lease>> {
|
||||
|
||||
@@ -23,6 +23,17 @@ pub mod layout;
|
||||
pub mod v016;
|
||||
|
||||
pub async fn try_migrate(server: &Server) -> trc::Result<()> {
|
||||
migrate(server, layout::Run::Boot).await.map(|_| ())
|
||||
}
|
||||
|
||||
/// inbuxa: K-S3: `--upgrade-store`. What a start would do to the store
|
||||
/// before serving, and nothing else; another node's upgrade is refused
|
||||
/// rather than waited for.
|
||||
pub async fn upgrade_store(server: &Server) -> trc::Result<layout::Outcome> {
|
||||
migrate(server, layout::Run::UpgradeOnly).await
|
||||
}
|
||||
|
||||
async fn migrate(server: &Server, run: layout::Run) -> trc::Result<layout::Outcome> {
|
||||
// inbuxa: a store a newer release has changed is refused before anything
|
||||
// below writes to it.
|
||||
let store_version = layout::store_version(server.store()).await?;
|
||||
@@ -47,7 +58,7 @@ pub async fn try_migrate(server: &Server) -> trc::Result<()> {
|
||||
{
|
||||
Some(DATABASE_SCHEMA_VERSION) => {
|
||||
if !std::env::var("DANGER_FORCE_MIGRATE").is_ok_and(|v| v == "1") {
|
||||
return layout::upgrade_layout(server, store_version, false).await;
|
||||
return layout::upgrade_layout(server, store_version, false, run).await;
|
||||
}
|
||||
}
|
||||
Some(0..=4) => {
|
||||
@@ -74,7 +85,7 @@ pub async fn try_migrate(server: &Server) -> trc::Result<()> {
|
||||
_ => {
|
||||
if is_new_install(server).await.caused_by(trc::location!())? {
|
||||
write_schema_version(server).await?;
|
||||
return layout::upgrade_layout(server, store_version, true).await;
|
||||
return layout::upgrade_layout(server, store_version, true, run).await;
|
||||
} else {
|
||||
abort(concat!(
|
||||
"You must first upgrade to version 0.15, please read ",
|
||||
@@ -86,7 +97,7 @@ pub async fn try_migrate(server: &Server) -> trc::Result<()> {
|
||||
|
||||
migrate_v0_16(server).await?;
|
||||
write_schema_version(server).await?;
|
||||
layout::upgrade_layout(server, store_version, false).await
|
||||
layout::upgrade_layout(server, store_version, false, run).await
|
||||
}
|
||||
|
||||
async fn write_schema_version(server: &Server) -> trc::Result<()> {
|
||||
|
||||
@@ -94,6 +94,7 @@ async fn store_version() {
|
||||
|
||||
layout_step_1_2(server).await;
|
||||
layout_lease(server).await;
|
||||
upgrade_store_only(server).await;
|
||||
layout_step_2_3(server).await;
|
||||
}
|
||||
|
||||
@@ -245,15 +246,14 @@ async fn write_lease(server: &Server, node_id: u16, expires_at: u64) {
|
||||
}
|
||||
|
||||
async fn lease_held(server: &Server) -> bool {
|
||||
server
|
||||
.store()
|
||||
.get_value::<u64>(AnyKey {
|
||||
subspace: SUBSPACE_INBUXA,
|
||||
key: migration::layout::LAYOUT_LEASE_KEY.to_vec(),
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
.is_some()
|
||||
// A lease is a node id and an expiry, so it's read as it is
|
||||
get_raw(
|
||||
server.store(),
|
||||
SUBSPACE_INBUXA,
|
||||
migration::layout::LAYOUT_LEASE_KEY,
|
||||
)
|
||||
.await
|
||||
.is_some()
|
||||
}
|
||||
|
||||
// SL-3: a node that can't take the lease waits for the mark; a lease that
|
||||
@@ -309,6 +309,57 @@ async fn layout_lease(server: &Server) {
|
||||
assert!(!lease_held(server).await, "lease not released");
|
||||
}
|
||||
|
||||
// inbuxa: K-S3: `--upgrade-store` upgrades and stops; it changes nothing on
|
||||
// a current store, and refuses, rather than waits, while another node holds
|
||||
// the lease.
|
||||
async fn upgrade_store_only(server: &Server) {
|
||||
use migration::layout::Outcome;
|
||||
|
||||
// Current: nothing to do, twice over
|
||||
write_mark(server, Some(INBUXA_STORE_VERSION)).await;
|
||||
for _ in 0..2 {
|
||||
assert_eq!(
|
||||
migration::upgrade_store(server).await.unwrap(),
|
||||
Outcome::Current
|
||||
);
|
||||
assert_eq!(read_mark(server).await, Some(INBUXA_STORE_VERSION));
|
||||
}
|
||||
|
||||
// Another node holds the lease: refused at once, and its lease is left
|
||||
write_mark(server, Some(1)).await;
|
||||
write_lease(server, 900, now() + 600).await;
|
||||
let outcome = tokio::time::timeout(Duration::from_secs(5), migration::upgrade_store(server))
|
||||
.await
|
||||
.expect("waited on another node's lease instead of refusing")
|
||||
.unwrap();
|
||||
match outcome {
|
||||
Outcome::Refused(reason) => assert!(reason.contains("node 900"), "{reason}"),
|
||||
other => panic!("expected a refusal, got {other:?}"),
|
||||
}
|
||||
assert_eq!(
|
||||
read_mark(server).await,
|
||||
Some(1),
|
||||
"upgraded under another lease"
|
||||
);
|
||||
assert!(
|
||||
lease_held(server).await,
|
||||
"the other node's lease was removed"
|
||||
);
|
||||
|
||||
// Once it's gone: upgraded, lease released, and a second run is a no-op
|
||||
write_lease(server, 900, now() - 1).await;
|
||||
assert_eq!(
|
||||
migration::upgrade_store(server).await.unwrap(),
|
||||
Outcome::Upgraded { from: 1 }
|
||||
);
|
||||
assert_eq!(read_mark(server).await, Some(INBUXA_STORE_VERSION));
|
||||
assert!(!lease_held(server).await, "lease not released");
|
||||
assert_eq!(
|
||||
migration::upgrade_store(server).await.unwrap(),
|
||||
Outcome::Current
|
||||
);
|
||||
}
|
||||
|
||||
async fn applications(server: &Server) -> Vec<(String, bool)> {
|
||||
let mut apps = server
|
||||
.registry()
|
||||
|
||||
Reference in new issue
Block a user