diff --git a/.gitea/workflows/announce.yml b/.gitea/workflows/announce.yml index db2d2b4a..59d5bf0b 100644 --- a/.gitea/workflows/announce.yml +++ b/.gitea/workflows/announce.yml @@ -11,7 +11,7 @@ jobs: announce: runs-on: light steps: - - uses: coffey-labs/actions/discourse-release@ba4cde7c3b47cc9f93c0080da8ba838a1ef1845b + - uses: coffey-labs/actions/discourse-release@958b3a35a345744abd236ca5abf679cbedaca7a0 with: api-key: ${{ secrets.DISCOURSE_RELEASE_KEY }} discord-webhook: ${{ secrets.DISCORD_RELEASE_WEBHOOK }} diff --git a/.gitea/workflows/publish.yml b/.gitea/workflows/publish.yml index 30b162b0..b17ca664 100644 --- a/.gitea/workflows/publish.yml +++ b/.gitea/workflows/publish.yml @@ -511,7 +511,7 @@ jobs: if: ${{ always() && ((needs.release.result == 'success' && needs.binaries.result == 'success') || needs.github.result == 'success') }} runs-on: light steps: - - uses: coffey-labs/actions/discourse-release@ba4cde7c3b47cc9f93c0080da8ba838a1ef1845b + - uses: coffey-labs/actions/discourse-release@958b3a35a345744abd236ca5abf679cbedaca7a0 with: api-key: ${{ secrets.DISCOURSE_RELEASE_KEY }} discord-webhook: ${{ secrets.DISCORD_RELEASE_WEBHOOK }} diff --git a/crates/common/src/manager/backup.rs b/crates/common/src/manager/backup.rs index c7bdddc4..c0722075 100644 --- a/crates/common/src/manager/backup.rs +++ b/crates/common/src/manager/backup.rs @@ -11,7 +11,7 @@ use ahash::AHashSet; use lz4_flex::frame::FrameEncoder; use std::{ io::{BufWriter, Write}, - path::{Path, PathBuf}, + path::PathBuf, sync::mpsc::{self, SyncSender}, }; use store::{ @@ -19,7 +19,7 @@ use store::{ *, }; use types::blob_hash::{BLOB_HASH_LEN, BlobHash}; -use utils::{UnwrapFailure, codec::leb128::Leb128_}; +use utils::{UnwrapFailure, codec::leb128::Leb128_, failed}; pub(super) const MAGIC_MARKER: u8 = 123; @@ -52,9 +52,31 @@ pub struct BackupParams { families: AHashSet, } +/// inbuxa: K-C11: where the export files go +enum Sink { + Dir(PathBuf), + S3(store::inbuxa_transfer::S3Transfer), +} + +type Upload = tokio::task::JoinHandle>; + +/// inbuxa: K-C11: an export in S3 ends with this, written last, naming its +/// files: an export without one didn't finish. +pub const S3_EXPORT_MANIFEST: &str = "export.json"; + impl Core { pub async fn backup(&self, mut params: BackupParams) { - if !params.dest.exists() { + // inbuxa: K-C11: s3://bucket/prefix exports to a new folder there + let dest_text = params.dest.to_string_lossy().to_string(); + let sink = if store::inbuxa_transfer::is_s3_url(&dest_text) { + let transfer = store::inbuxa_transfer::S3Transfer::open(&dest_text) + .unwrap_or_else(|err| failed(&format!("Export failed: {err}"))); + Sink::S3(transfer.join(&store::inbuxa_transfer::export_name(store::write::now()))) + } else { + Sink::Dir(params.dest.clone()) + }; + if let Sink::S3(_) = &sink { + } else if !params.dest.exists() { std::fs::create_dir_all(¶ms.dest).failed("Failed to create backup directory"); } else if !params.dest.is_dir() { eprintln!("Backup destination {:?} is not a directory.", params.dest); @@ -73,7 +95,9 @@ impl Core { .failed("Could not retrieve database schema version.") .failed("Could not retrieve database schema version."); + let mut families_given = true; if params.families.is_empty() { + families_given = false; params.families = [ Family::Data, Family::Registry, @@ -89,34 +113,81 @@ impl Core { .collect(); } - for subspace in params - .families - .into_iter() - .flat_map(|f| f.subspaces()) - .copied() - { + // inbuxa: K-C11: blobs last. An export taken while the server runs + // then holds every blob the rest of it names: a message stored + // during the export is either in it whole or not at all. (In a set's + // order, the blobs could go first, and the messages stored after + // them would point at blobs the export never got.) + let mut families = params.families.into_iter().collect::>(); + families.sort_by_key(|f| (*f == Family::Blob, *f as u8)); + + let mut uploads: Vec<(String, u8, Upload)> = Vec::new(); + for subspace in families.into_iter().flat_map(|f| f.subspaces()).copied() { + let name = format!("subspace_{}", store::subspace_name(subspace)); + let (writer, upload) = match &sink { + Sink::Dir(dir) => (Output::File(dir.join(&name)), None), + Sink::S3(transfer) => { + println!("Exporting database to {}/{name}.", transfer.location()); + let (writer, upload) = transfer.writer(&name); + (Output::S3(writer), Some(upload)) + } + }; let (async_handle, sync_handle) = if subspace == SUBSPACE_BLOBS { - self.backup_blobs(¶ms.dest, subspace, schema_version) + self.backup_blobs(writer, subspace, schema_version) } else { - self.backup_subspace(¶ms.dest, subspace, schema_version) + self.backup_subspace(writer, subspace, schema_version) }; async_handle.await.failed("Task failed"); sync_handles.push(sync_handle); + if let Some(upload) = upload { + uploads.push((name, subspace, upload)); + } } for handle in sync_handles { handle.join().expect("Failed to join thread"); } + + // inbuxa: K-C11: the uploads, then the manifest that says the + // export is whole + if let Sink::S3(transfer) = &sink { + let mut files = Vec::new(); + let mut bytes = 0u64; + for (name, subspace, upload) in uploads { + let size = upload + .await + .failed("Upload task failed") + .unwrap_or_else(|err| failed(&format!("Export failed: {err}"))); + bytes += size; + files.push(serde_json::json!({"name": name, "subspace": subspace, "bytes": size})); + } + let manifest = serde_json::json!({ + "format": 1, + "version": types::brand_version!(), + "schemaVersion": schema_version, + "createdAt": store::write::now(), + "partial": families_given, + "files": files, + }); + transfer + .put( + S3_EXPORT_MANIFEST, + &serde_json::to_vec_pretty(&manifest).unwrap_or_default(), + ) + .await + .unwrap_or_else(|err| failed(&format!("Export failed: {err}"))); + println!( + "Exported {} files, {bytes} bytes compressed, to {}.", + manifest["files"].as_array().map_or(0, Vec::len), + transfer.location() + ); + } } - fn backup_blobs(&self, dest: &Path, subspace: u8, schema_version: u32) -> TaskHandle { + fn backup_blobs(&self, output: Output, subspace: u8, schema_version: u32) -> TaskHandle { let store = self.storage.data.clone(); let blob_store = self.storage.blob.clone(); - let (handle, writer) = spawn_writer( - dest.join(format!("subspace_{}", store::subspace_name(subspace))), - subspace, - schema_version, - ); + let (handle, writer) = spawn_writer(output, subspace, schema_version); ( tokio::spawn(async move { let mut blobs = Vec::new(); @@ -175,13 +246,9 @@ impl Core { ) } - fn backup_subspace(&self, dest: &Path, subspace: u8, schema_version: u32) -> TaskHandle { + fn backup_subspace(&self, output: Output, subspace: u8, schema_version: u32) -> TaskHandle { let store = self.storage.data.clone(); - let (handle, writer) = spawn_writer( - dest.join(format!("subspace_{}", store::subspace_name(subspace))), - subspace, - schema_version, - ); + let (handle, writer) = spawn_writer(output, subspace, schema_version); ( tokio::spawn(async move { if !store.is_sql() || (subspace != SUBSPACE_COUNTER && subspace != SUBSPACE_QUOTA) { @@ -253,20 +320,60 @@ impl Core { } } +/// inbuxa: K-C11: one export file's destination +enum Output { + File(PathBuf), + S3(store::inbuxa_transfer::PartWriter), +} + +/// inbuxa: K-C11: the sending end of an export file's writer. The export's +/// tasks send from the runtime's workers; when the writer is behind (an +/// upload to S3 waits on the runtime too), a plain blocking send would hold +/// the worker it needs, and with one worker (a pod limited to one CPU) the +/// export would stop there. A send that has to wait hands its worker over +/// first. +struct ExportSender(SyncSender<(Vec, Vec)>); + +impl ExportSender { + fn send(&self, item: (Vec, Vec)) -> Result<(), mpsc::SendError<(Vec, Vec)>> { + match self.0.try_send(item) { + Ok(()) => Ok(()), + Err(mpsc::TrySendError::Full(item)) => { + if can_block_in_place() { + tokio::task::block_in_place(|| self.0.send(item)) + } else { + self.0.send(item) + } + } + Err(mpsc::TrySendError::Disconnected(item)) => Err(mpsc::SendError(item)), + } + } +} + +/// Whether this thread is a worker of a multi-threaded runtime, the only +/// kind `block_in_place` works on +fn can_block_in_place() -> bool { + tokio::runtime::Handle::try_current() + .is_ok_and(|h| h.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread) +} + #[allow(clippy::type_complexity)] fn spawn_writer( - path: PathBuf, + output: Output, subspace: u8, version: u32, -) -> (std::thread::JoinHandle<()>, SyncSender<(Vec, Vec)>) { +) -> (std::thread::JoinHandle<()>, ExportSender) { let (tx, rx) = mpsc::sync_channel::<(Vec, Vec)>(10); let handle = std::thread::spawn(move || { - println!("Exporting database to {}.", path.to_str().unwrap()); - - let mut file = FrameEncoder::new(BufWriter::new( - std::fs::File::create(path).failed("Failed to create backup file"), - )); + let out: Box = match output { + Output::File(path) => { + println!("Exporting database to {}.", path.to_str().unwrap()); + Box::new(std::fs::File::create(path).failed("Failed to create backup file")) + } + Output::S3(writer) => Box::new(writer), + }; + let mut file = FrameEncoder::new(BufWriter::new(out)); file.write_all(&[MAGIC_MARKER, subspace]) .failed("Failed to write version"); file.write_all(&version.to_le_bytes()) @@ -290,7 +397,7 @@ fn spawn_writer( file.flush().failed("Failed to flush backup file"); }); - (handle, tx) + (handle, ExportSender(tx)) } impl BackupParams { diff --git a/crates/common/src/manager/boot.rs b/crates/common/src/manager/boot.rs index 02308570..76ed2e8d 100644 --- a/crates/common/src/manager/boot.rs +++ b/crates/common/src/manager/boot.rs @@ -42,6 +42,9 @@ pub struct BootManager { /// inbuxa: K-S8: whether the store was new when this process started, /// before the defaults went in pub store_was_new: bool, + /// inbuxa: K-C11: started with `--store-check quick|full`: like + /// `--upgrade-plan`, then main runs a store check here and exits + pub store_check: Option, } pub struct IpcReceivers { @@ -61,13 +64,14 @@ Usage: inbuxa [OPTIONS] Options: -c, --config Start server with the specified configuration file - -e, --export Export all store data to a specific path - -i, --import Import store data from a specific path + -e, --export Export all store data to a directory or s3://bucket/prefix + -i, --import Import store data from a directory or s3:// address -o, --console Open the store console -u, --upgrade-store Upgrade the store's layout, then exit --seed Set up a new store from a file, then exit --upgrade-plan Print what --upgrade-store would do, then exit --check Check the data, blob store and coordinator, then exit + --store-check Check every blob the store names, then exit -h, --help Print help -V, --version Print version @@ -78,6 +82,20 @@ queues the index to be rebuilt when the server next starts. EXPORT_TYPES limits an export to some of: data, registry, blob, changelog, queue, report, telemetry, tasks. +An export to s3://bucket/prefix streams to a new folder there, named for the +time (20261119T204500Z), and writes export.json last: an export without it +didn't finish. An import from s3://bucket/prefix/ reads one back; from +s3://bucket/prefix, the newest finished one. INBUXA_EXPORT_S3_ENDPOINT names +an S3-compatible service (otherwise AWS), INBUXA_EXPORT_S3_REGION its region, +and INBUXA_EXPORT_S3_ACCESS_KEY_FILE and INBUXA_EXPORT_S3_SECRET_KEY_FILE +files holding its keys (otherwise the AWS_ variables or the instance's role). + +--store-check runs a store check, as the console's Store checks page does: +quick finds blobs that are missing, full also reads each one and checks its +hash. It is saved with the others, and prints a summary. It exits 0 when it +found nothing wrong, 3 when it found problems (the console lists them), +2 when another check was running, 1 when it failed. + --upgrade-store does what a start does to the store before it serves, and exits: 0 when the store is at this release's layout (whether it upgraded it or found it there), 1 when the upgrade failed, 2 when it refused because @@ -119,6 +137,8 @@ enum StoreOp { UpgradePlan, // inbuxa: K-C5 Check, + // inbuxa: K-C11 + StoreCheck(String), None, } @@ -150,7 +170,7 @@ impl BootManager { }; let takes_value = matches!( key.as_str(), - "config" | "c" | "export" | "e" | "import" | "i" | "seed" + "config" | "c" | "export" | "e" | "import" | "i" | "seed" | "store-check" ); let value = if takes_value { inline.or_else(|| args.next()) @@ -191,6 +211,10 @@ impl BootManager { ("seed", Some(value)) => { import_export = StoreOp::Seed(value.into()); } + ("store-check", Some(value)) => match value.as_str() { + "quick" | "full" => import_export = StoreOp::StoreCheck(value), + _ => failed("--store-check takes quick or full."), + }, (_, None) if takes_value => failed(&format!( "Missing value for argument '{key}', try '--help'." )), @@ -223,7 +247,11 @@ impl BootManager { }; // inbuxa: K-S8: a seed run boots as --upgrade-store does let upgrade_plan = import_export == StoreOp::UpgradePlan; - let one_shot = upgrade_store || upgrade_plan || seed.is_some(); + let store_check = match &import_export { + StoreOp::StoreCheck(mode) => Some(mode.clone()), + _ => None, + }; + let one_shot = upgrade_store || upgrade_plan || seed.is_some() || store_check.is_some(); let registry = RegistryStore::init( PathBuf::from(config_path.unwrap()), import_export == StoreOp::None || one_shot, @@ -232,8 +260,9 @@ impl BootManager { .unwrap_or_else(|err| startup_failed(err)); if one_shot && (registry.is_bootstrap_mode() || registry.is_recovery_mode()) { failed( - "--upgrade-store and --seed need a configured server: there is no data \ - store in bootstrap mode, and recovery mode is for a person at the console.", + "--upgrade-store, --seed and --store-check need a configured server: there is \ + no data store in bootstrap mode, and recovery mode is for a person at the \ + console.", ); } let mut bootstrap = Bootstrap::new(registry).await; @@ -267,7 +296,11 @@ impl BootManager { let telemetry = Telemetry::parse(&mut bootstrap, &storage).await; match import_export { - StoreOp::None | StoreOp::UpgradeStore | StoreOp::UpgradePlan | StoreOp::Seed(_) => { + StoreOp::None + | StoreOp::UpgradeStore + | StoreOp::UpgradePlan + | StoreOp::Seed(_) + | StoreOp::StoreCheck(_) => { // Parse components let core: Box = Box::new(Box::pin(Core::parse(&mut bootstrap, storage)).await); @@ -285,6 +318,8 @@ impl BootManager { "Started with --upgrade-store: upgrading the store layout, then exiting" } else if upgrade_plan { "Started with --upgrade-plan: reporting what an upgrade would do, then exiting" + } else if store_check.is_some() { + "Started with --store-check: checking the store's blobs, then exiting" } else { "Started with --seed: setting up a new store from a file, then exiting" }, @@ -381,6 +416,7 @@ impl BootManager { upgrade_plan, seed, store_was_new, + store_check, } } StoreOp::Export(path) => { diff --git a/crates/common/src/manager/restore.rs b/crates/common/src/manager/restore.rs index 99b37800..cbf6c4c4 100644 --- a/crates/common/src/manager/restore.rs +++ b/crates/common/src/manager/restore.rs @@ -6,7 +6,7 @@ * Modified by Coffey Labs in 2026 for INBUXA. */ -use super::backup::MAGIC_MARKER; +use super::backup::{MAGIC_MARKER, S3_EXPORT_MANIFEST}; use crate::{Core, DATABASE_SCHEMA_VERSION}; use lz4_flex::frame::FrameDecoder; use registry::{ @@ -22,7 +22,7 @@ use std::{ path::{Path, PathBuf}, }; use store::{ - BlobStore, IterateParams, SUBSPACE_BLOBS, SUBSPACE_COUNTER, SUBSPACE_INDEXES, + BlobStore, IterateParams, SUBSPACE_BLOBS, SUBSPACE_COUNTER, SUBSPACE_INBUXA, SUBSPACE_INDEXES, SUBSPACE_PROPERTY, SUBSPACE_QUOTA, SUBSPACE_REGISTRY_PK, SUBSPACE_TELEMETRY_SPAN, Store, U32_LEN, write::{ @@ -37,6 +37,12 @@ impl Core { /// Imports an export into an empty store and returns the subspaces it /// wrote. inbuxa: the caller hands them to [`Core::queue_reindex`]. pub async fn restore(&self, src: PathBuf) -> Vec { + // inbuxa: K-C11: an export in S3 + let src_text = src.to_string_lossy().to_string(); + if store::inbuxa_transfer::is_s3_url(&src_text) { + return self.restore_s3(&src_text).await; + } + // Backup the core let paths = if src.is_dir() { let mut paths = Vec::new(); @@ -52,11 +58,91 @@ impl Core { vec![src] }; + let sources = paths.into_iter().map(Source::File).collect::>(); + self.restore_sources(sources).await + } + + /// inbuxa: K-C11: `s3://bucket/prefix/`, or the newest finished + /// export under `s3://bucket/prefix`. An export without its manifest + /// didn't finish, and is refused. + async fn restore_s3(&self, url: &str) -> Vec { + use store::inbuxa_transfer::S3Transfer; + let base = + S3Transfer::open(url).unwrap_or_else(|err| failed(&format!("Import failed: {err}"))); + let (transfer, manifest) = match base.get(S3_EXPORT_MANIFEST).await { + Ok(Some(manifest)) => (base, manifest), + Ok(None) => { + let names = base + .children_with(S3_EXPORT_MANIFEST) + .await + .unwrap_or_else(|err| failed(&format!("Import failed: {err}"))); + let Some(newest) = names.last() else { + failed(&format!( + "Import failed: no finished export at {url} or under it (an export is \ + finished once its {S3_EXPORT_MANIFEST} is written)." + )); + }; + let transfer = base.join(newest); + println!( + "Importing the newest finished export, {}.", + transfer.location() + ); + let manifest = transfer + .get(S3_EXPORT_MANIFEST) + .await + .ok() + .flatten() + .unwrap_or_else(|| failed("Import failed: the export's manifest went away")); + (transfer, manifest) + } + Err(err) => failed(&format!("Import failed: {err}")), + }; + let manifest: serde_json::Value = serde_json::from_slice(&manifest) + .unwrap_or_else(|err| failed(&format!("Import failed: {S3_EXPORT_MANIFEST}: {err}"))); + let sources = manifest["files"] + .as_array() + .unwrap_or_else(|| { + failed(&format!( + "Import failed: {S3_EXPORT_MANIFEST} lists no files" + )) + }) + .iter() + .map(|file| { + let name = file["name"].as_str().unwrap_or_default(); + let subspace = file["subspace"].as_u64().and_then(|s| u8::try_from(s).ok()); + match (name, subspace) { + (name, Some(subspace)) + if name.starts_with("subspace_") && !name.contains('/') => + { + Source::S3 { + transfer: transfer.clone(), + name: name.to_string(), + subspace, + } + } + _ => failed(&format!( + "Import failed: {S3_EXPORT_MANIFEST}: a bad file entry: {file}" + )), + } + }) + .collect::>(); + self.restore_sources(sources).await + } + + async fn restore_sources(&self, paths: Vec) -> Vec { let mut conflicts = Vec::new(); for path in &paths { - let subspace = KeyValueReader::new(path).subspace; + let subspace = path.subspace(); + // inbuxa: K-C11: every start writes the fork's own subspace, this + // import's too: where the blob store is (W), the permission + // grants (P), the log files' setting (T). Its records are keyed + // values, which an import sets over, never duplicates; a store in + // use still has data in the other subspaces, which are checked. + if subspace == SUBSPACE_INBUXA { + continue; + } if subspace_has_data(&self.storage.data, subspace).await { - conflicts.push(path.clone()); + conflicts.push(path.describe()); } } @@ -68,34 +154,85 @@ impl Core { start the server before importing. Conflicting dumps:" ); for path in conflicts { - eprintln!(" {}", path.display()); + eprintln!(" {path}"); } std::process::exit(1); } - let mut imported = paths - .iter() - .map(|path| KeyValueReader::new(path).subspace) - .collect::>(); + let mut imported = paths.iter().map(Source::subspace).collect::>(); imported.sort_unstable(); imported.dedup(); + // inbuxa: K-C11: the blobs last, into the blob store the imported + // settings name. This process started on an empty store, whose + // blob store is the data store; a server started on the imported + // one reads blobs from wherever its x:BlobStore says (S3, say), so + // blobs written to the data store would be missing there. + let (blobs, rest): (Vec<_>, Vec<_>) = paths + .into_iter() + .partition(|path| path.subspace() == SUBSPACE_BLOBS); + let mut tasks = Vec::new(); - for path in paths { + for path in rest { let storage = self.storage.clone(); let blob_store = self.storage.blob.clone(); tasks.push(tokio::spawn(async move { restore_file(storage.data, blob_store, &path).await; })); } - for task in tasks { task.await.failed("Failed to wait for task"); } + if !blobs.is_empty() { + let blob_store = self.imported_blob_store().await; + let mut tasks = Vec::new(); + for path in blobs { + let storage = self.storage.clone(); + let blob_store = blob_store.clone(); + tasks.push(tokio::spawn(async move { + restore_file(storage.data, blob_store, &path).await; + })); + } + for task in tasks { + task.await.failed("Failed to wait for task"); + } + } + imported } + /// inbuxa: K-C11: the blob store the store's x:BlobStore names now, + /// after the import wrote it + async fn imported_blob_store(&self) -> BlobStore { + use registry::{schema::structs::BlobStore as BlobStoreSetting, types::ObjectImpl}; + let setting = self + .storage + .registry + .object::(BlobStoreSetting::OBJECT.singleton().id()) + .await + .failed("Failed to read the imported blob store setting") + .unwrap_or_default(); + if matches!(setting, BlobStoreSetting::Default) { + println!("Importing blobs into the data store, as the export's settings say."); + return self.storage.data.clone().into(); + } + let mut warnings = Vec::new(); + let store = BlobStore::open(setting, &self.storage.data, &mut warnings) + .await + .unwrap_or_else(|err| { + failed(&format!( + "Can't open the blob store the export's settings name, to import the \ + blobs into: {err}" + )) + }); + for warning in warnings { + eprintln!("Blob store: {warning}"); + } + println!("Importing blobs into the blob store the export's settings name."); + store + } + /// inbuxa: an export never carries the full-text index. It is built by /// and for one search backend (the SQL stores index into their own /// tables, the key-value stores into a subspace, external engines keep it @@ -172,10 +309,59 @@ async fn subspace_has_data(store: &Store, subspace: u8) -> bool { has_data } -async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { - println!("Importing database dump from {}.", path.to_str().unwrap()); +/// inbuxa: K-C11: one export file: on disk, or in S3 +enum Source { + File(PathBuf), + S3 { + transfer: store::inbuxa_transfer::S3Transfer, + name: String, + /// From the manifest, so the file needn't be opened to know it + subspace: u8, + }, +} - let mut reader = KeyValueReader::new(path); +impl Source { + fn subspace(&self) -> u8 { + match self { + Source::File(path) => KeyValueReader::new(path).subspace, + Source::S3 { subspace, .. } => *subspace, + } + } + + fn describe(&self) -> String { + match self { + Source::File(path) => path.display().to_string(), + Source::S3 { transfer, name, .. } => format!("{}/{name}", transfer.location()), + } + } + + fn open(&self) -> KeyValueReader { + match self { + Source::File(path) => KeyValueReader::new(path), + Source::S3 { + transfer, + name, + subspace, + } => { + let reader = + KeyValueReader::from_reader(Box::new(transfer.reader(name)), &self.describe()); + if reader.subspace != *subspace { + failed(&format!( + "{} holds subspace {}, but the manifest says {subspace}", + self.describe(), + reader.subspace + )); + } + reader + } + } + } +} + +async fn restore_file(store: Store, blob_store: BlobStore, path: &Source) { + println!("Importing database dump from {}.", path.describe()); + + let mut reader = path.open(); let mut batch = BatchBuilder::new(); match reader.subspace { @@ -276,14 +462,20 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { struct KeyValueReader { subspace: u8, - file: FrameDecoder>, + file: FrameDecoder>>, } impl KeyValueReader { fn new(path: &Path) -> Self { - let mut file = FrameDecoder::new(BufReader::new( - File::open(path).failed("Failed to open file"), - )); + Self::from_reader( + Box::new(File::open(path).failed("Failed to open file")), + &path.display().to_string(), + ) + } + + // inbuxa: K-C11: any reader: a file, or a download from S3 + fn from_reader(reader: Box, path: &str) -> Self { + let mut file = FrameDecoder::new(BufReader::new(reader)); let mut buf = [0u8; 1]; file.read_exact(&mut buf) .failed(&format!("Failed to read magic marker from {path:?}")); diff --git a/crates/main/src/main.rs b/crates/main/src/main.rs index d4a05c6d..66f37fde 100644 --- a/crates/main/src/main.rs +++ b/crates/main/src/main.rs @@ -80,6 +80,12 @@ async fn main() -> std::io::Result<()> { std::process::exit(Box::pin(seed_and_exit(&server, &path, init.store_was_new)).await); } + // inbuxa: K-C11: --store-check runs a check and exits + if let Some(mode) = init.store_check.clone() { + let server = init.inner.build_server(); + std::process::exit(Box::pin(store_check_and_exit(&server, &mode)).await); + } + // Migrate database if let Err(err) = migration::try_migrate(&init.inner.build_server()).await { trc::event!( @@ -274,6 +280,60 @@ async fn seed_and_exit( flush_and_exit_code(code).await } +/// inbuxa: K-C11: `--store-check`: 0 found nothing wrong, 3 found problems, +/// 2 another check was running, 1 failed +async fn store_check_and_exit(server: &common::Server, mode: &str) -> i32 { + use services::inbuxa_store_check::{ + CheckMode as Mode, CheckRefusal as Refusal, CheckStatus as Status, + }; + + let mode = Mode::parse(mode).unwrap_or(Mode::Quick); + let (code, message) = match services::inbuxa_store_check::run_now(server, mode).await { + Ok(Ok(check)) => { + let p = &check.progress; + let summary = format!( + "Store check {} ({}): {} blobs, {} links, {} messages checked, {} bytes read. \ + Missing {}, damaged {}, unlinked {}, unreadable {}.", + check.id, + mode.as_str(), + p.blobs_checked, + p.links_checked, + p.messages_checked, + p.bytes_read, + p.missing, + p.damaged, + p.unlinked, + p.unreadable, + ); + match check.status { + Status::Done if p.problems() == 0 => (0, format!("{summary} Nothing wrong.")), + Status::Done => ( + 3, + format!( + "{summary} {} problems: the console's Store checks page lists them.", + p.problems() + ), + ), + status => ( + 1, + format!( + "{summary} It ended {}: {}", + status.as_str(), + check.error.as_deref().unwrap_or("no reason given") + ), + ), + } + } + Ok(Err(Refusal::Running(id))) => ( + 2, + format!("Store check {id} is running: one check runs at a time."), + ), + Err(err) => (1, format!("The store check failed: {err}")), + }; + println!("{message}"); + flush_and_exit_code(code).await +} + /// inbuxa: K-C9: `--upgrade-plan`: one line of JSON on stdout, for a /// script to read, and the exit code a shell can branch on async fn upgrade_plan_and_exit(server: &common::Server) -> i32 { diff --git a/crates/services/src/inbuxa_store_check.rs b/crates/services/src/inbuxa_store_check.rs index 5486d3c6..eca13d3c 100644 --- a/crates/services/src/inbuxa_store_check.rs +++ b/crates/services/src/inbuxa_store_check.rs @@ -74,6 +74,35 @@ pub async fn start( Ok(Ok(id)) } +// inbuxa: K-C11: for main's --store-check +pub use inbuxa_features::store_check::{ + Mode as CheckMode, Refusal as CheckRefusal, Status as CheckStatus, +}; + +/// inbuxa: K-C11: the node id a `--store-check` run records. Leased node +/// ids are 16-bit, so no server's start-up (SV-8) takes it for its own. +pub const ONE_SHOT_NODE_ID: u64 = 1 << 32; + +/// inbuxa: K-C11: `--store-check`: a check run here, to its end, for a +/// scheduled job. Saved like any other, so the console shows it. +pub async fn run_now(server: &Server, mode: Mode) -> trc::Result> { + let now = now(); + let check = StoreCheck::new(mode, "store-check job".to_string(), ONE_SHOT_NODE_ID, now); + let id = match model::create(server.store(), &check, now).await? { + Ok(id) => id, + Err(refusal) => return Ok(Err(refusal)), + }; + run(server, StoreCheck { id, ..check }).await; + Ok(model::get(server.store(), id) + .await? + .ok_or_else(|| { + trc::StoreEvent::NotFound + .into_err() + .details("The check went away") + }) + .map(Ok)?) +} + /// SV-8: asks a running check to stop. Any node can: the one running it /// sees the request at its next save. None if there's no such check. pub async fn stop(server: &Server, id: u32) -> trc::Result> { diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index 566e2a78..f509c565 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -65,7 +65,7 @@ foundation = ["foundationdb", "futures"] fdb-chunked-bm = [] # Blob stores -s3 = ["rust-s3", "dep:reqwest_s3"] +s3 = ["rust-s3", "dep:reqwest_s3", "futures"] azure = ["azure_core", "azure_storage", "azure_storage_blobs", "futures"] # In-memory stores diff --git a/crates/store/src/inbuxa_transfer.rs b/crates/store/src/inbuxa_transfer.rs new file mode 100644 index 00000000..4d5f1952 --- /dev/null +++ b/crates/store/src/inbuxa_transfer.rs @@ -0,0 +1,592 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +//! inbuxa: K-C11: an export written to, and read back from, an S3 bucket. +//! +//! `--export s3://bucket/prefix` streams each export file into a multipart +//! upload, so nothing the size of the store lands on local disk, and +//! `--import s3://bucket/prefix/` streams it back. The bucket's +//! address and keys come from the environment: +//! +//! - `INBUXA_EXPORT_S3_ENDPOINT`: an S3-compatible service (MinIO, Garage, +//! Ceph...), addressed path-style. Without it, AWS. +//! - `INBUXA_EXPORT_S3_REGION`: `us-east-1` unless given. +//! - `INBUXA_EXPORT_S3_ACCESS_KEY_FILE`, `INBUXA_EXPORT_S3_SECRET_KEY_FILE`: +//! files holding the keys. Without them, the usual AWS sources (the +//! `AWS_` variables, a profile, the instance's role). +//! +//! The export and import code is sync (a writer thread per file, readers in +//! the import tasks); these bridge it to the async S3 client. + +/// Whether a `--export`/`--import` value names a bucket. +pub fn is_s3_url(value: &str) -> bool { + value.starts_with("s3://") +} + +/// The name an export made now goes under: `20261119T204500Z`. +pub fn export_name(now: u64) -> String { + let days = (now / 86_400) as i64; + let secs = now % 86_400; + // Howard Hinnant's civil_from_days + let z = days + 719_468; + let era = z.div_euclid(146_097); + let doe = z.rem_euclid(146_097); + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let day = doy - (153 * mp + 2) / 5 + 1; + let month = if mp < 10 { mp + 3 } else { mp - 9 }; + let year = yoe + era * 400 + i64::from(month <= 2); + format!( + "{year:04}{month:02}{day:02}T{:02}{:02}{:02}Z", + secs / 3600, + (secs / 60) % 60, + secs % 60 + ) +} + +/// The size of upload part `n` (from 1): 16 MiB for the first thousand, +/// doubling every thousand after, up to 512 MiB. S3 takes at most 10,000 +/// parts, so one file can be nearly 3 TiB, and only one part is held at a +/// time. +pub fn part_size(part: u32) -> usize { + const MIB: usize = 1024 * 1024; + let tier = (part.saturating_sub(1) / 1000).min(5); + (16 * MIB) << tier +} + +/// The most parts S3 takes in one upload. +pub const MAX_PARTS: u32 = 10_000; + +#[cfg(feature = "s3")] +pub use imp::{PartReader, PartWriter, S3Transfer}; + +#[cfg(not(feature = "s3"))] +pub use none::{PartReader, PartWriter, S3Transfer}; + +#[cfg(feature = "s3")] +mod imp { + use super::{MAX_PARTS, part_size}; + use futures::StreamExt; + use s3::{Bucket, Region, creds::Credentials, serde_types::Part}; + use std::io::{Read, Write}; + use std::sync::mpsc as std_mpsc; + use tokio::sync::mpsc; + + const CONTENT_TYPE: &str = "application/octet-stream"; + const RETRIES: u32 = 5; + + #[derive(Clone)] + pub struct S3Transfer { + bucket: Box, + bucket_name: String, + /// No leading or trailing slash; may be empty. + prefix: String, + } + + fn env(name: &str) -> Option { + std::env::var(name) + .ok() + .map(|v| v.trim().to_string()) + .filter(|v| !v.is_empty()) + } + + fn key_file(name: &str) -> Result, String> { + match env(name) { + Some(path) => std::fs::read_to_string(&path) + .map(|v| Some(v.trim_end_matches(['\r', '\n']).to_string())) + .map_err(|err| format!("{name}: can't read {path}: {err}")), + None => Ok(None), + } + } + + impl S3Transfer { + /// `s3://bucket/prefix`, with the bucket's address and keys from the + /// environment. + pub fn open(url: &str) -> Result { + let rest = url + .strip_prefix("s3://") + .ok_or_else(|| format!("{url}: not an s3:// address"))?; + let (bucket_name, prefix) = rest.split_once('/').unwrap_or((rest, "")); + if bucket_name.is_empty() { + return Err(format!("{url}: no bucket named")); + } + let region_name = env("INBUXA_EXPORT_S3_REGION").unwrap_or_else(|| "us-east-1".into()); + let endpoint = env("INBUXA_EXPORT_S3_ENDPOINT"); + let region = match &endpoint { + Some(endpoint) => Region::Custom { + region: region_name, + endpoint: endpoint.trim_end_matches('/').to_string(), + }, + None => region_name + .parse() + .map_err(|err| format!("INBUXA_EXPORT_S3_REGION: {err}"))?, + }; + let access = key_file("INBUXA_EXPORT_S3_ACCESS_KEY_FILE")?; + let secret = key_file("INBUXA_EXPORT_S3_SECRET_KEY_FILE")?; + let credentials = match (&access, &secret) { + (Some(access), Some(secret)) => { + Credentials::new(Some(access), Some(secret), None, None, None) + } + (None, None) => Credentials::default(), + _ => { + return Err("INBUXA_EXPORT_S3_ACCESS_KEY_FILE and \ + INBUXA_EXPORT_S3_SECRET_KEY_FILE go together" + .into()); + } + } + .map_err(|err| format!("S3 credentials: {err}"))?; + let mut bucket = Bucket::new(bucket_name, region, credentials) + .map_err(|err| format!("S3 bucket {bucket_name}: {err}"))?; + if endpoint.is_some() { + bucket = bucket.with_path_style(); + } + Ok(S3Transfer { + bucket, + bucket_name: bucket_name.to_string(), + prefix: prefix.trim_matches('/').to_string(), + }) + } + + /// The same bucket, one level down. + pub fn join(&self, name: &str) -> Self { + S3Transfer { + prefix: self.key(name), + ..self.clone() + } + } + + pub fn location(&self) -> String { + format!("s3://{}/{}", self.bucket_name, self.prefix) + } + + fn key(&self, name: &str) -> String { + if self.prefix.is_empty() { + name.to_string() + } else { + format!("{}/{name}", self.prefix) + } + } + + /// A small object, whole. + pub async fn put(&self, name: &str, data: &[u8]) -> Result<(), String> { + let key = self.key(name); + let mut last = String::new(); + for attempt in 0..RETRIES { + match self.bucket.put_object(&key, data).await { + Ok(r) if (200..300).contains(&r.status_code()) => return Ok(()), + Ok(r) => { + last = format!( + "{}: {}", + r.status_code(), + String::from_utf8_lossy(r.as_slice()) + ) + } + Err(err) => last = err.to_string(), + } + wait(attempt).await; + } + Err(format!( + "Failed to write {}: {last}", + self.location_of(name) + )) + } + + /// A small object, whole; None when it isn't there. + pub async fn get(&self, name: &str) -> Result>, String> { + let key = self.key(name); + let mut last = String::new(); + for attempt in 0..RETRIES { + match self.bucket.get_object(&key).await { + Ok(r) if (200..300).contains(&r.status_code()) => { + return Ok(Some(r.to_vec())); + } + Ok(r) if r.status_code() == 404 => return Ok(None), + Ok(r) => last = format!("{}", r.status_code()), + Err(s3::error::S3Error::HttpFailWithBody(404, _)) => return Ok(None), + Err(err) => last = err.to_string(), + } + wait(attempt).await; + } + Err(format!("Failed to read {}: {last}", self.location_of(name))) + } + + /// The names one level down that hold `marker`: the exports under a + /// prefix. + pub async fn children_with(&self, marker: &str) -> Result, String> { + let prefix = if self.prefix.is_empty() { + String::new() + } else { + format!("{}/", self.prefix) + }; + let results = self + .bucket + .list(prefix.clone(), None) + .await + .map_err(|err| format!("Failed to list {}: {err}", self.location()))?; + let mut names = results + .iter() + .flat_map(|r| r.contents.iter()) + .filter_map(|o| o.key.strip_prefix(&prefix)) + .filter_map(|rest| rest.strip_suffix(marker)?.strip_suffix('/')) + .filter(|name| !name.is_empty() && !name.contains('/')) + .map(str::to_string) + .collect::>(); + names.sort(); + Ok(names) + } + + fn location_of(&self, name: &str) -> String { + format!("s3://{}/{}", self.bucket_name, self.key(name)) + } + + /// A writer whose bytes go up as a multipart upload under `name`, + /// for a thread outside the runtime; the upload runs on the + /// runtime. Dropping the writer sends the last part; the handle + /// ends when the upload is complete. + pub fn writer( + &self, + name: &str, + ) -> (PartWriter, tokio::task::JoinHandle>) { + let (tx, rx) = mpsc::channel::>(1); + let this = self.clone(); + let name = name.to_string(); + let handle = tokio::spawn(async move { this.upload(&name, rx).await }); + ( + PartWriter { + tx: Some(tx), + buf: Vec::with_capacity(part_size(1)), + part: 1, + }, + handle, + ) + } + + async fn upload(&self, name: &str, mut rx: mpsc::Receiver>) -> Result { + let key = self.key(name); + let location = self.location_of(name); + let upload_id = self + .bucket + .initiate_multipart_upload(&key, CONTENT_TYPE) + .await + .map_err(|err| format!("Failed to start uploading {location}: {err}"))? + .upload_id; + let mut parts: Vec = Vec::new(); + let mut total = 0u64; + let result: Result<(), String> = async { + while let Some(chunk) = rx.recv().await { + let number = parts.len() as u32 + 1; + if number > MAX_PARTS { + return Err(format!("{location}: too large for one upload")); + } + total += chunk.len() as u64; + parts.push( + self.put_part(chunk, &key, number, &upload_id, &location) + .await?, + ); + } + Ok(()) + } + .await; + if let Err(err) = result { + // Dropping rx fails the writer's next send, which stops it + drop(rx); + let _ = self.bucket.abort_upload(&key, &upload_id).await; + return Err(err); + } + let mut last = String::new(); + for attempt in 0..RETRIES { + match self + .bucket + .complete_multipart_upload(&key, &upload_id, parts.clone()) + .await + { + Ok(r) if (200..300).contains(&r.status_code()) => return Ok(total), + Ok(r) => last = format!("{}", r.status_code()), + Err(err) => last = err.to_string(), + } + wait(attempt).await; + } + let _ = self.bucket.abort_upload(&key, &upload_id).await; + Err(format!("Failed to finish uploading {location}: {last}")) + } + + async fn put_part( + &self, + chunk: Vec, + key: &str, + number: u32, + upload_id: &str, + location: &str, + ) -> Result { + let mut last = String::new(); + for attempt in 0..RETRIES { + match self + .bucket + .put_multipart_chunk(chunk.clone(), key, number, upload_id, CONTENT_TYPE) + .await + { + Ok(part) => return Ok(part), + Err(err) => last = err.to_string(), + } + wait(attempt).await; + } + Err(format!( + "Failed to upload part {number} of {location}: {last}" + )) + } + + /// A reader of the object `name`, for code that reads synchronously + /// on the runtime's workers: the download is a task on the runtime, + /// and a reader that has to wait for it hands its worker over. + pub fn reader(&self, name: &str) -> PartReader { + let (tx, rx) = std_mpsc::sync_channel::, String>>(4); + let this = self.clone(); + let key = self.key(name); + let location = self.location_of(name); + tokio::spawn(async move { + { + let mut stream = match this.bucket.get_object_stream(&key).await { + Ok(stream) if (200..300).contains(&stream.status_code) => stream, + Ok(stream) => { + let _ = send_chunk( + &tx, + Err(format!("Failed to read {location}: {}", stream.status_code)), + ); + return; + } + Err(err) => { + let _ = + send_chunk(&tx, Err(format!("Failed to read {location}: {err}"))); + return; + } + }; + while let Some(chunk) = stream.bytes().next().await { + let sent = match chunk { + Ok(bytes) => send_chunk(&tx, Ok(bytes.to_vec())), + Err(err) => { + let _ = send_chunk( + &tx, + Err(format!("Failed to read {location}: {err}")), + ); + return; + } + }; + if sent.is_err() { + return; + } + } + } + }); + PartReader { + rx, + buf: Vec::new(), + pos: 0, + } + } + } + + /// Whether this thread is a worker of a multi-threaded runtime, where a + /// wait can hand the worker over (`block_in_place`) + fn on_worker() -> bool { + tokio::runtime::Handle::try_current() + .is_ok_and(|h| h.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread) + } + + /// A send that, when it has to wait for the reader, hands the worker over + fn send_chunk( + tx: &std_mpsc::SyncSender, String>>, + item: Result, String>, + ) -> Result<(), ()> { + match tx.try_send(item) { + Ok(()) => Ok(()), + Err(std_mpsc::TrySendError::Disconnected(_)) => Err(()), + Err(std_mpsc::TrySendError::Full(item)) if on_worker() => { + tokio::task::block_in_place(|| tx.send(item)).map_err(|_| ()) + } + Err(std_mpsc::TrySendError::Full(item)) => tx.send(item).map_err(|_| ()), + } + } + + async fn wait(attempt: u32) { + tokio::time::sleep(std::time::Duration::from_millis(250 << attempt.min(5))).await; + } + + /// See [`S3Transfer::writer`]. + pub struct PartWriter { + tx: Option>>, + buf: Vec, + part: u32, + } + + impl PartWriter { + fn send(&mut self) -> std::io::Result<()> { + let next = part_size(self.part + 1); + let chunk = std::mem::replace(&mut self.buf, Vec::with_capacity(next)); + self.part += 1; + match &self.tx { + Some(tx) => tx.blocking_send(chunk).map_err(|_| { + std::io::Error::other("the upload stopped: its error is on the next line") + }), + None => Err(std::io::Error::other("the upload is finished")), + } + } + } + + impl Write for PartWriter { + fn write(&mut self, data: &[u8]) -> std::io::Result { + let room = part_size(self.part) - self.buf.len(); + let take = room.min(data.len()); + self.buf.extend_from_slice(&data[..take]); + if self.buf.len() == part_size(self.part) { + self.send()?; + } + Ok(take) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + impl Drop for PartWriter { + fn drop(&mut self) { + // The last part, which may be short; even an empty file is one + // part + if !self.buf.is_empty() || self.part == 1 { + let _ = self.send(); + } + self.tx = None; + } + } + + /// See [`S3Transfer::reader`]. + pub struct PartReader { + rx: std_mpsc::Receiver, String>>, + buf: Vec, + pos: usize, + } + + impl Read for PartReader { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + while self.pos == self.buf.len() { + // Readers run on the runtime's workers (the import's tasks), + // and the download needs one to make progress: a wait hands + // this worker over first, or a pod with one CPU's worth of + // workers would stop here + let next = match self.rx.try_recv() { + Ok(next) => Ok(next), + Err(std_mpsc::TryRecvError::Disconnected) => Err(std_mpsc::RecvError), + Err(std_mpsc::TryRecvError::Empty) => { + if on_worker() { + tokio::task::block_in_place(|| self.rx.recv()) + } else { + self.rx.recv() + } + } + }; + match next { + Ok(Ok(chunk)) => { + self.buf = chunk; + self.pos = 0; + } + Ok(Err(err)) => return Err(std::io::Error::other(err)), + // The download ended + Err(_) => return Ok(0), + } + } + let n = out.len().min(self.buf.len() - self.pos); + out[..n].copy_from_slice(&self.buf[self.pos..self.pos + n]); + self.pos += n; + Ok(n) + } + } +} + +#[cfg(not(feature = "s3"))] +mod none { + //! This build has no S3 client: [`S3Transfer::open`] always refuses, so + //! nothing else here is ever reached. + + #[derive(Clone)] + pub struct S3Transfer; + pub struct PartWriter; + pub struct PartReader; + + impl S3Transfer { + pub fn open(_url: &str) -> Result { + Err("this build has no S3 support (the s3 feature)".into()) + } + pub fn join(&self, _name: &str) -> Self { + unreachable!() + } + pub fn location(&self) -> String { + unreachable!() + } + pub async fn put(&self, _name: &str, _data: &[u8]) -> Result<(), String> { + unreachable!() + } + pub async fn get(&self, _name: &str) -> Result>, String> { + unreachable!() + } + pub async fn children_with(&self, _marker: &str) -> Result, String> { + unreachable!() + } + pub fn writer( + &self, + _name: &str, + ) -> (PartWriter, tokio::task::JoinHandle>) { + unreachable!() + } + pub fn reader(&self, _name: &str) -> PartReader { + unreachable!() + } + } + + impl std::io::Write for PartWriter { + fn write(&mut self, _data: &[u8]) -> std::io::Result { + unreachable!() + } + fn flush(&mut self) -> std::io::Result<()> { + unreachable!() + } + } + + impl std::io::Read for PartReader { + fn read(&mut self, _out: &mut [u8]) -> std::io::Result { + unreachable!() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn export_names_sort_by_time() { + assert_eq!(export_name(0), "19700101T000000Z"); + assert_eq!(export_name(1_795_552_496), "20261124T203456Z"); + assert_eq!(export_name(951_782_400), "20000229T000000Z"); + assert!(export_name(1_795_552_496) < export_name(1_795_552_497)); + } + + #[test] + fn parts_grow_to_nearly_three_tebibytes() { + assert_eq!(part_size(1), 16 << 20); + assert_eq!(part_size(1000), 16 << 20); + assert_eq!(part_size(1001), 32 << 20); + assert_eq!(part_size(MAX_PARTS), 512 << 20); + let total: u64 = (1..=MAX_PARTS).map(|n| part_size(n) as u64).sum(); + assert!(total > (29u64 << 40) / 10, "{total}"); + } + + #[test] + fn s3_urls() { + assert!(is_s3_url("s3://backups/inbuxa")); + assert!(!is_s3_url("/var/backups")); + } +} diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 440c0595..a7b69d2b 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -11,6 +11,8 @@ pub mod backend; pub mod build; pub mod dispatch; +// inbuxa: K-C11: exports to and from S3 +pub mod inbuxa_transfer; pub mod query; pub mod registry; pub mod search; diff --git a/tests/src/store/import_export.rs b/tests/src/store/import_export.rs index 20d72aa3..f94d23a7 100644 --- a/tests/src/store/import_export.rs +++ b/tests/src/store/import_export.rs @@ -278,6 +278,31 @@ pub async fn test(test: &TestServer) { .backup(BackupParams::new(temp_dir.path.clone())) .await; + // inbuxa: K-C11: a record in the fork's subspace, which every start + // writes (here whichever one the test server wrote first) + let mut startup_key = None; + db.iterate( + IterateParams::new( + AnyKey { + subspace: SUBSPACE_INBUXA, + key: vec![0u8], + }, + AnyKey { + subspace: SUBSPACE_INBUXA, + key: vec![u8::MAX; 32], + }, + ) + .no_values(), + |key, _| { + startup_key = Some(key.to_vec()); + Ok(false) + }, + ) + .await + .unwrap(); + let startup_key = + startup_key.expect("the test server's start wrote nothing to the fork's subspace"); + // Destroy store println!("Destroying store..."); store_destroy(&db).await; @@ -293,6 +318,16 @@ pub async fn test(test: &TestServer) { .write("localhost") .finalize(), ); + // inbuxa: K-C11: and over what any start writes first, the import's own + // too: a record in the fork's subspace, with a value the import has to + // set over. Refused as data in the store until rc.34. + batch.set( + ValueClass::Any(AnyClass { + subspace: SUBSPACE_INBUXA, + key: startup_key, + }), + b"written at start-up".to_vec(), + ); db.write(batch.build_all()).await.unwrap(); for (key, _) in &named_blobs { test.server.blob_store().delete_blob(key).await.unwrap();