From 4f2187b805c6b5ba6c4f1dcfb4bf37c6a5fe310b Mon Sep 17 00:00:00 2001 From: John Coffey Date: Thu, 19 Nov 2026 13:47:43 -0700 Subject: [PATCH] Backups for Kubernetes: export to and import from S3, store checks as a command (K-C11) For the chart's scheduled backup and store checks (inbuxa-charts#11), and three bugs the lab's restore test found on the way. Added - `--export s3://bucket/prefix` streams the export into multipart uploads, in a new folder named for the time (20261119T204500Z), and writes export.json last, naming the files: an export without it didn't finish. Nothing the size of the store lands on local disk. Parts start at 16 MiB and double every thousand up to 512 MiB, so one file can be nearly 3 TiB; one part is held at a time. - `--import s3://bucket/prefix/` reads one back, streamed; from `s3://bucket/prefix`, the newest finished one. - The bucket from INBUXA_EXPORT_S3_ENDPOINT (S3-compatible, path-style; AWS without it), INBUXA_EXPORT_S3_REGION, and the keys from INBUXA_EXPORT_S3_ACCESS_KEY_FILE and INBUXA_EXPORT_S3_SECRET_KEY_FILE (the AWS defaults without them). - `--store-check quick|full` runs a store check to its end and exits: 0 nothing wrong, 3 problems found, 2 another check running, 1 failed. It's saved with the console's other checks, under a node id no server holds. Fixed - A live export could name blobs it didn't have: the store's areas went out in a set's order, so the blobs could go first and a message stored after them pointed at a blob the export never got. Blobs go last now. - An import into a fresh store refused to run: every start writes the fork's subspace (where the blob store is, permission grants, the log files' setting), the import's own start too, and the import took that for data. Since the blob store change (#333): 27.1-rc.16 and 2026.11.9.1. The fork's subspace is no longer counted; its records are keyed values an import sets over. The store test now imports over a start-up record, and fails without this. - An import wrote the blobs into the data store, the empty store's blob store, even when the imported settings name another (S3, say): the restored server then found none of them. Blobs are imported last, into the blob store the imported x:BlobStore names. - The S3 export and import could stop for good on a pod limited to one CPU (one runtime worker): a reader or sender waiting on the transfer held the worker the transfer needed. A wait hands its worker over first. Tested: inbuxa-k8s-lab backup-test.sh, on the kind lab (Postgres, Garage, NATS, five server pods): mail for an account, the chart's backup CronJob, the database dropped and the blob bucket emptied, the import, and every mailbox's count and message hashes the same after; quick and full store checks clean. store tests pass on SQLite and RocksDB; unit tests pass. --- .gitea/workflows/announce.yml | 2 +- .gitea/workflows/publish.yml | 2 +- crates/common/src/manager/backup.rs | 169 ++++-- crates/common/src/manager/boot.rs | 50 +- crates/common/src/manager/restore.rs | 228 ++++++++- crates/main/src/main.rs | 60 +++ crates/services/src/inbuxa_store_check.rs | 29 ++ crates/store/Cargo.toml | 2 +- crates/store/src/inbuxa_transfer.rs | 592 ++++++++++++++++++++++ crates/store/src/lib.rs | 2 + tests/src/store/import_export.rs | 35 ++ 11 files changed, 1112 insertions(+), 59 deletions(-) create mode 100644 crates/store/src/inbuxa_transfer.rs diff --git a/.gitea/workflows/announce.yml b/.gitea/workflows/announce.yml index db2d2b4a..59d5bf0b 100644 --- a/.gitea/workflows/announce.yml +++ b/.gitea/workflows/announce.yml @@ -11,7 +11,7 @@ jobs: announce: runs-on: light steps: - - uses: coffey-labs/actions/discourse-release@ba4cde7c3b47cc9f93c0080da8ba838a1ef1845b + - uses: coffey-labs/actions/discourse-release@958b3a35a345744abd236ca5abf679cbedaca7a0 with: api-key: ${{ secrets.DISCOURSE_RELEASE_KEY }} discord-webhook: ${{ secrets.DISCORD_RELEASE_WEBHOOK }} diff --git a/.gitea/workflows/publish.yml b/.gitea/workflows/publish.yml index 30b162b0..b17ca664 100644 --- a/.gitea/workflows/publish.yml +++ b/.gitea/workflows/publish.yml @@ -511,7 +511,7 @@ jobs: if: ${{ always() && ((needs.release.result == 'success' && needs.binaries.result == 'success') || needs.github.result == 'success') }} runs-on: light steps: - - uses: coffey-labs/actions/discourse-release@ba4cde7c3b47cc9f93c0080da8ba838a1ef1845b + - uses: coffey-labs/actions/discourse-release@958b3a35a345744abd236ca5abf679cbedaca7a0 with: api-key: ${{ secrets.DISCOURSE_RELEASE_KEY }} discord-webhook: ${{ secrets.DISCORD_RELEASE_WEBHOOK }} diff --git a/crates/common/src/manager/backup.rs b/crates/common/src/manager/backup.rs index c7bdddc4..c0722075 100644 --- a/crates/common/src/manager/backup.rs +++ b/crates/common/src/manager/backup.rs @@ -11,7 +11,7 @@ use ahash::AHashSet; use lz4_flex::frame::FrameEncoder; use std::{ io::{BufWriter, Write}, - path::{Path, PathBuf}, + path::PathBuf, sync::mpsc::{self, SyncSender}, }; use store::{ @@ -19,7 +19,7 @@ use store::{ *, }; use types::blob_hash::{BLOB_HASH_LEN, BlobHash}; -use utils::{UnwrapFailure, codec::leb128::Leb128_}; +use utils::{UnwrapFailure, codec::leb128::Leb128_, failed}; pub(super) const MAGIC_MARKER: u8 = 123; @@ -52,9 +52,31 @@ pub struct BackupParams { families: AHashSet, } +/// inbuxa: K-C11: where the export files go +enum Sink { + Dir(PathBuf), + S3(store::inbuxa_transfer::S3Transfer), +} + +type Upload = tokio::task::JoinHandle>; + +/// inbuxa: K-C11: an export in S3 ends with this, written last, naming its +/// files: an export without one didn't finish. +pub const S3_EXPORT_MANIFEST: &str = "export.json"; + impl Core { pub async fn backup(&self, mut params: BackupParams) { - if !params.dest.exists() { + // inbuxa: K-C11: s3://bucket/prefix exports to a new folder there + let dest_text = params.dest.to_string_lossy().to_string(); + let sink = if store::inbuxa_transfer::is_s3_url(&dest_text) { + let transfer = store::inbuxa_transfer::S3Transfer::open(&dest_text) + .unwrap_or_else(|err| failed(&format!("Export failed: {err}"))); + Sink::S3(transfer.join(&store::inbuxa_transfer::export_name(store::write::now()))) + } else { + Sink::Dir(params.dest.clone()) + }; + if let Sink::S3(_) = &sink { + } else if !params.dest.exists() { std::fs::create_dir_all(¶ms.dest).failed("Failed to create backup directory"); } else if !params.dest.is_dir() { eprintln!("Backup destination {:?} is not a directory.", params.dest); @@ -73,7 +95,9 @@ impl Core { .failed("Could not retrieve database schema version.") .failed("Could not retrieve database schema version."); + let mut families_given = true; if params.families.is_empty() { + families_given = false; params.families = [ Family::Data, Family::Registry, @@ -89,34 +113,81 @@ impl Core { .collect(); } - for subspace in params - .families - .into_iter() - .flat_map(|f| f.subspaces()) - .copied() - { + // inbuxa: K-C11: blobs last. An export taken while the server runs + // then holds every blob the rest of it names: a message stored + // during the export is either in it whole or not at all. (In a set's + // order, the blobs could go first, and the messages stored after + // them would point at blobs the export never got.) + let mut families = params.families.into_iter().collect::>(); + families.sort_by_key(|f| (*f == Family::Blob, *f as u8)); + + let mut uploads: Vec<(String, u8, Upload)> = Vec::new(); + for subspace in families.into_iter().flat_map(|f| f.subspaces()).copied() { + let name = format!("subspace_{}", store::subspace_name(subspace)); + let (writer, upload) = match &sink { + Sink::Dir(dir) => (Output::File(dir.join(&name)), None), + Sink::S3(transfer) => { + println!("Exporting database to {}/{name}.", transfer.location()); + let (writer, upload) = transfer.writer(&name); + (Output::S3(writer), Some(upload)) + } + }; let (async_handle, sync_handle) = if subspace == SUBSPACE_BLOBS { - self.backup_blobs(¶ms.dest, subspace, schema_version) + self.backup_blobs(writer, subspace, schema_version) } else { - self.backup_subspace(¶ms.dest, subspace, schema_version) + self.backup_subspace(writer, subspace, schema_version) }; async_handle.await.failed("Task failed"); sync_handles.push(sync_handle); + if let Some(upload) = upload { + uploads.push((name, subspace, upload)); + } } for handle in sync_handles { handle.join().expect("Failed to join thread"); } + + // inbuxa: K-C11: the uploads, then the manifest that says the + // export is whole + if let Sink::S3(transfer) = &sink { + let mut files = Vec::new(); + let mut bytes = 0u64; + for (name, subspace, upload) in uploads { + let size = upload + .await + .failed("Upload task failed") + .unwrap_or_else(|err| failed(&format!("Export failed: {err}"))); + bytes += size; + files.push(serde_json::json!({"name": name, "subspace": subspace, "bytes": size})); + } + let manifest = serde_json::json!({ + "format": 1, + "version": types::brand_version!(), + "schemaVersion": schema_version, + "createdAt": store::write::now(), + "partial": families_given, + "files": files, + }); + transfer + .put( + S3_EXPORT_MANIFEST, + &serde_json::to_vec_pretty(&manifest).unwrap_or_default(), + ) + .await + .unwrap_or_else(|err| failed(&format!("Export failed: {err}"))); + println!( + "Exported {} files, {bytes} bytes compressed, to {}.", + manifest["files"].as_array().map_or(0, Vec::len), + transfer.location() + ); + } } - fn backup_blobs(&self, dest: &Path, subspace: u8, schema_version: u32) -> TaskHandle { + fn backup_blobs(&self, output: Output, subspace: u8, schema_version: u32) -> TaskHandle { let store = self.storage.data.clone(); let blob_store = self.storage.blob.clone(); - let (handle, writer) = spawn_writer( - dest.join(format!("subspace_{}", store::subspace_name(subspace))), - subspace, - schema_version, - ); + let (handle, writer) = spawn_writer(output, subspace, schema_version); ( tokio::spawn(async move { let mut blobs = Vec::new(); @@ -175,13 +246,9 @@ impl Core { ) } - fn backup_subspace(&self, dest: &Path, subspace: u8, schema_version: u32) -> TaskHandle { + fn backup_subspace(&self, output: Output, subspace: u8, schema_version: u32) -> TaskHandle { let store = self.storage.data.clone(); - let (handle, writer) = spawn_writer( - dest.join(format!("subspace_{}", store::subspace_name(subspace))), - subspace, - schema_version, - ); + let (handle, writer) = spawn_writer(output, subspace, schema_version); ( tokio::spawn(async move { if !store.is_sql() || (subspace != SUBSPACE_COUNTER && subspace != SUBSPACE_QUOTA) { @@ -253,20 +320,60 @@ impl Core { } } +/// inbuxa: K-C11: one export file's destination +enum Output { + File(PathBuf), + S3(store::inbuxa_transfer::PartWriter), +} + +/// inbuxa: K-C11: the sending end of an export file's writer. The export's +/// tasks send from the runtime's workers; when the writer is behind (an +/// upload to S3 waits on the runtime too), a plain blocking send would hold +/// the worker it needs, and with one worker (a pod limited to one CPU) the +/// export would stop there. A send that has to wait hands its worker over +/// first. +struct ExportSender(SyncSender<(Vec, Vec)>); + +impl ExportSender { + fn send(&self, item: (Vec, Vec)) -> Result<(), mpsc::SendError<(Vec, Vec)>> { + match self.0.try_send(item) { + Ok(()) => Ok(()), + Err(mpsc::TrySendError::Full(item)) => { + if can_block_in_place() { + tokio::task::block_in_place(|| self.0.send(item)) + } else { + self.0.send(item) + } + } + Err(mpsc::TrySendError::Disconnected(item)) => Err(mpsc::SendError(item)), + } + } +} + +/// Whether this thread is a worker of a multi-threaded runtime, the only +/// kind `block_in_place` works on +fn can_block_in_place() -> bool { + tokio::runtime::Handle::try_current() + .is_ok_and(|h| h.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread) +} + #[allow(clippy::type_complexity)] fn spawn_writer( - path: PathBuf, + output: Output, subspace: u8, version: u32, -) -> (std::thread::JoinHandle<()>, SyncSender<(Vec, Vec)>) { +) -> (std::thread::JoinHandle<()>, ExportSender) { let (tx, rx) = mpsc::sync_channel::<(Vec, Vec)>(10); let handle = std::thread::spawn(move || { - println!("Exporting database to {}.", path.to_str().unwrap()); - - let mut file = FrameEncoder::new(BufWriter::new( - std::fs::File::create(path).failed("Failed to create backup file"), - )); + let out: Box = match output { + Output::File(path) => { + println!("Exporting database to {}.", path.to_str().unwrap()); + Box::new(std::fs::File::create(path).failed("Failed to create backup file")) + } + Output::S3(writer) => Box::new(writer), + }; + let mut file = FrameEncoder::new(BufWriter::new(out)); file.write_all(&[MAGIC_MARKER, subspace]) .failed("Failed to write version"); file.write_all(&version.to_le_bytes()) @@ -290,7 +397,7 @@ fn spawn_writer( file.flush().failed("Failed to flush backup file"); }); - (handle, tx) + (handle, ExportSender(tx)) } impl BackupParams { diff --git a/crates/common/src/manager/boot.rs b/crates/common/src/manager/boot.rs index 02308570..76ed2e8d 100644 --- a/crates/common/src/manager/boot.rs +++ b/crates/common/src/manager/boot.rs @@ -42,6 +42,9 @@ pub struct BootManager { /// inbuxa: K-S8: whether the store was new when this process started, /// before the defaults went in pub store_was_new: bool, + /// inbuxa: K-C11: started with `--store-check quick|full`: like + /// `--upgrade-plan`, then main runs a store check here and exits + pub store_check: Option, } pub struct IpcReceivers { @@ -61,13 +64,14 @@ Usage: inbuxa [OPTIONS] Options: -c, --config Start server with the specified configuration file - -e, --export Export all store data to a specific path - -i, --import Import store data from a specific path + -e, --export Export all store data to a directory or s3://bucket/prefix + -i, --import Import store data from a directory or s3:// address -o, --console Open the store console -u, --upgrade-store Upgrade the store's layout, then exit --seed Set up a new store from a file, then exit --upgrade-plan Print what --upgrade-store would do, then exit --check Check the data, blob store and coordinator, then exit + --store-check Check every blob the store names, then exit -h, --help Print help -V, --version Print version @@ -78,6 +82,20 @@ queues the index to be rebuilt when the server next starts. EXPORT_TYPES limits an export to some of: data, registry, blob, changelog, queue, report, telemetry, tasks. +An export to s3://bucket/prefix streams to a new folder there, named for the +time (20261119T204500Z), and writes export.json last: an export without it +didn't finish. An import from s3://bucket/prefix/ reads one back; from +s3://bucket/prefix, the newest finished one. INBUXA_EXPORT_S3_ENDPOINT names +an S3-compatible service (otherwise AWS), INBUXA_EXPORT_S3_REGION its region, +and INBUXA_EXPORT_S3_ACCESS_KEY_FILE and INBUXA_EXPORT_S3_SECRET_KEY_FILE +files holding its keys (otherwise the AWS_ variables or the instance's role). + +--store-check runs a store check, as the console's Store checks page does: +quick finds blobs that are missing, full also reads each one and checks its +hash. It is saved with the others, and prints a summary. It exits 0 when it +found nothing wrong, 3 when it found problems (the console lists them), +2 when another check was running, 1 when it failed. + --upgrade-store does what a start does to the store before it serves, and exits: 0 when the store is at this release's layout (whether it upgraded it or found it there), 1 when the upgrade failed, 2 when it refused because @@ -119,6 +137,8 @@ enum StoreOp { UpgradePlan, // inbuxa: K-C5 Check, + // inbuxa: K-C11 + StoreCheck(String), None, } @@ -150,7 +170,7 @@ impl BootManager { }; let takes_value = matches!( key.as_str(), - "config" | "c" | "export" | "e" | "import" | "i" | "seed" + "config" | "c" | "export" | "e" | "import" | "i" | "seed" | "store-check" ); let value = if takes_value { inline.or_else(|| args.next()) @@ -191,6 +211,10 @@ impl BootManager { ("seed", Some(value)) => { import_export = StoreOp::Seed(value.into()); } + ("store-check", Some(value)) => match value.as_str() { + "quick" | "full" => import_export = StoreOp::StoreCheck(value), + _ => failed("--store-check takes quick or full."), + }, (_, None) if takes_value => failed(&format!( "Missing value for argument '{key}', try '--help'." )), @@ -223,7 +247,11 @@ impl BootManager { }; // inbuxa: K-S8: a seed run boots as --upgrade-store does let upgrade_plan = import_export == StoreOp::UpgradePlan; - let one_shot = upgrade_store || upgrade_plan || seed.is_some(); + let store_check = match &import_export { + StoreOp::StoreCheck(mode) => Some(mode.clone()), + _ => None, + }; + let one_shot = upgrade_store || upgrade_plan || seed.is_some() || store_check.is_some(); let registry = RegistryStore::init( PathBuf::from(config_path.unwrap()), import_export == StoreOp::None || one_shot, @@ -232,8 +260,9 @@ impl BootManager { .unwrap_or_else(|err| startup_failed(err)); if one_shot && (registry.is_bootstrap_mode() || registry.is_recovery_mode()) { failed( - "--upgrade-store and --seed need a configured server: there is no data \ - store in bootstrap mode, and recovery mode is for a person at the console.", + "--upgrade-store, --seed and --store-check need a configured server: there is \ + no data store in bootstrap mode, and recovery mode is for a person at the \ + console.", ); } let mut bootstrap = Bootstrap::new(registry).await; @@ -267,7 +296,11 @@ impl BootManager { let telemetry = Telemetry::parse(&mut bootstrap, &storage).await; match import_export { - StoreOp::None | StoreOp::UpgradeStore | StoreOp::UpgradePlan | StoreOp::Seed(_) => { + StoreOp::None + | StoreOp::UpgradeStore + | StoreOp::UpgradePlan + | StoreOp::Seed(_) + | StoreOp::StoreCheck(_) => { // Parse components let core: Box = Box::new(Box::pin(Core::parse(&mut bootstrap, storage)).await); @@ -285,6 +318,8 @@ impl BootManager { "Started with --upgrade-store: upgrading the store layout, then exiting" } else if upgrade_plan { "Started with --upgrade-plan: reporting what an upgrade would do, then exiting" + } else if store_check.is_some() { + "Started with --store-check: checking the store's blobs, then exiting" } else { "Started with --seed: setting up a new store from a file, then exiting" }, @@ -381,6 +416,7 @@ impl BootManager { upgrade_plan, seed, store_was_new, + store_check, } } StoreOp::Export(path) => { diff --git a/crates/common/src/manager/restore.rs b/crates/common/src/manager/restore.rs index 99b37800..cbf6c4c4 100644 --- a/crates/common/src/manager/restore.rs +++ b/crates/common/src/manager/restore.rs @@ -6,7 +6,7 @@ * Modified by Coffey Labs in 2026 for INBUXA. */ -use super::backup::MAGIC_MARKER; +use super::backup::{MAGIC_MARKER, S3_EXPORT_MANIFEST}; use crate::{Core, DATABASE_SCHEMA_VERSION}; use lz4_flex::frame::FrameDecoder; use registry::{ @@ -22,7 +22,7 @@ use std::{ path::{Path, PathBuf}, }; use store::{ - BlobStore, IterateParams, SUBSPACE_BLOBS, SUBSPACE_COUNTER, SUBSPACE_INDEXES, + BlobStore, IterateParams, SUBSPACE_BLOBS, SUBSPACE_COUNTER, SUBSPACE_INBUXA, SUBSPACE_INDEXES, SUBSPACE_PROPERTY, SUBSPACE_QUOTA, SUBSPACE_REGISTRY_PK, SUBSPACE_TELEMETRY_SPAN, Store, U32_LEN, write::{ @@ -37,6 +37,12 @@ impl Core { /// Imports an export into an empty store and returns the subspaces it /// wrote. inbuxa: the caller hands them to [`Core::queue_reindex`]. pub async fn restore(&self, src: PathBuf) -> Vec { + // inbuxa: K-C11: an export in S3 + let src_text = src.to_string_lossy().to_string(); + if store::inbuxa_transfer::is_s3_url(&src_text) { + return self.restore_s3(&src_text).await; + } + // Backup the core let paths = if src.is_dir() { let mut paths = Vec::new(); @@ -52,11 +58,91 @@ impl Core { vec![src] }; + let sources = paths.into_iter().map(Source::File).collect::>(); + self.restore_sources(sources).await + } + + /// inbuxa: K-C11: `s3://bucket/prefix/`, or the newest finished + /// export under `s3://bucket/prefix`. An export without its manifest + /// didn't finish, and is refused. + async fn restore_s3(&self, url: &str) -> Vec { + use store::inbuxa_transfer::S3Transfer; + let base = + S3Transfer::open(url).unwrap_or_else(|err| failed(&format!("Import failed: {err}"))); + let (transfer, manifest) = match base.get(S3_EXPORT_MANIFEST).await { + Ok(Some(manifest)) => (base, manifest), + Ok(None) => { + let names = base + .children_with(S3_EXPORT_MANIFEST) + .await + .unwrap_or_else(|err| failed(&format!("Import failed: {err}"))); + let Some(newest) = names.last() else { + failed(&format!( + "Import failed: no finished export at {url} or under it (an export is \ + finished once its {S3_EXPORT_MANIFEST} is written)." + )); + }; + let transfer = base.join(newest); + println!( + "Importing the newest finished export, {}.", + transfer.location() + ); + let manifest = transfer + .get(S3_EXPORT_MANIFEST) + .await + .ok() + .flatten() + .unwrap_or_else(|| failed("Import failed: the export's manifest went away")); + (transfer, manifest) + } + Err(err) => failed(&format!("Import failed: {err}")), + }; + let manifest: serde_json::Value = serde_json::from_slice(&manifest) + .unwrap_or_else(|err| failed(&format!("Import failed: {S3_EXPORT_MANIFEST}: {err}"))); + let sources = manifest["files"] + .as_array() + .unwrap_or_else(|| { + failed(&format!( + "Import failed: {S3_EXPORT_MANIFEST} lists no files" + )) + }) + .iter() + .map(|file| { + let name = file["name"].as_str().unwrap_or_default(); + let subspace = file["subspace"].as_u64().and_then(|s| u8::try_from(s).ok()); + match (name, subspace) { + (name, Some(subspace)) + if name.starts_with("subspace_") && !name.contains('/') => + { + Source::S3 { + transfer: transfer.clone(), + name: name.to_string(), + subspace, + } + } + _ => failed(&format!( + "Import failed: {S3_EXPORT_MANIFEST}: a bad file entry: {file}" + )), + } + }) + .collect::>(); + self.restore_sources(sources).await + } + + async fn restore_sources(&self, paths: Vec) -> Vec { let mut conflicts = Vec::new(); for path in &paths { - let subspace = KeyValueReader::new(path).subspace; + let subspace = path.subspace(); + // inbuxa: K-C11: every start writes the fork's own subspace, this + // import's too: where the blob store is (W), the permission + // grants (P), the log files' setting (T). Its records are keyed + // values, which an import sets over, never duplicates; a store in + // use still has data in the other subspaces, which are checked. + if subspace == SUBSPACE_INBUXA { + continue; + } if subspace_has_data(&self.storage.data, subspace).await { - conflicts.push(path.clone()); + conflicts.push(path.describe()); } } @@ -68,34 +154,85 @@ impl Core { start the server before importing. Conflicting dumps:" ); for path in conflicts { - eprintln!(" {}", path.display()); + eprintln!(" {path}"); } std::process::exit(1); } - let mut imported = paths - .iter() - .map(|path| KeyValueReader::new(path).subspace) - .collect::>(); + let mut imported = paths.iter().map(Source::subspace).collect::>(); imported.sort_unstable(); imported.dedup(); + // inbuxa: K-C11: the blobs last, into the blob store the imported + // settings name. This process started on an empty store, whose + // blob store is the data store; a server started on the imported + // one reads blobs from wherever its x:BlobStore says (S3, say), so + // blobs written to the data store would be missing there. + let (blobs, rest): (Vec<_>, Vec<_>) = paths + .into_iter() + .partition(|path| path.subspace() == SUBSPACE_BLOBS); + let mut tasks = Vec::new(); - for path in paths { + for path in rest { let storage = self.storage.clone(); let blob_store = self.storage.blob.clone(); tasks.push(tokio::spawn(async move { restore_file(storage.data, blob_store, &path).await; })); } - for task in tasks { task.await.failed("Failed to wait for task"); } + if !blobs.is_empty() { + let blob_store = self.imported_blob_store().await; + let mut tasks = Vec::new(); + for path in blobs { + let storage = self.storage.clone(); + let blob_store = blob_store.clone(); + tasks.push(tokio::spawn(async move { + restore_file(storage.data, blob_store, &path).await; + })); + } + for task in tasks { + task.await.failed("Failed to wait for task"); + } + } + imported } + /// inbuxa: K-C11: the blob store the store's x:BlobStore names now, + /// after the import wrote it + async fn imported_blob_store(&self) -> BlobStore { + use registry::{schema::structs::BlobStore as BlobStoreSetting, types::ObjectImpl}; + let setting = self + .storage + .registry + .object::(BlobStoreSetting::OBJECT.singleton().id()) + .await + .failed("Failed to read the imported blob store setting") + .unwrap_or_default(); + if matches!(setting, BlobStoreSetting::Default) { + println!("Importing blobs into the data store, as the export's settings say."); + return self.storage.data.clone().into(); + } + let mut warnings = Vec::new(); + let store = BlobStore::open(setting, &self.storage.data, &mut warnings) + .await + .unwrap_or_else(|err| { + failed(&format!( + "Can't open the blob store the export's settings name, to import the \ + blobs into: {err}" + )) + }); + for warning in warnings { + eprintln!("Blob store: {warning}"); + } + println!("Importing blobs into the blob store the export's settings name."); + store + } + /// inbuxa: an export never carries the full-text index. It is built by /// and for one search backend (the SQL stores index into their own /// tables, the key-value stores into a subspace, external engines keep it @@ -172,10 +309,59 @@ async fn subspace_has_data(store: &Store, subspace: u8) -> bool { has_data } -async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { - println!("Importing database dump from {}.", path.to_str().unwrap()); +/// inbuxa: K-C11: one export file: on disk, or in S3 +enum Source { + File(PathBuf), + S3 { + transfer: store::inbuxa_transfer::S3Transfer, + name: String, + /// From the manifest, so the file needn't be opened to know it + subspace: u8, + }, +} - let mut reader = KeyValueReader::new(path); +impl Source { + fn subspace(&self) -> u8 { + match self { + Source::File(path) => KeyValueReader::new(path).subspace, + Source::S3 { subspace, .. } => *subspace, + } + } + + fn describe(&self) -> String { + match self { + Source::File(path) => path.display().to_string(), + Source::S3 { transfer, name, .. } => format!("{}/{name}", transfer.location()), + } + } + + fn open(&self) -> KeyValueReader { + match self { + Source::File(path) => KeyValueReader::new(path), + Source::S3 { + transfer, + name, + subspace, + } => { + let reader = + KeyValueReader::from_reader(Box::new(transfer.reader(name)), &self.describe()); + if reader.subspace != *subspace { + failed(&format!( + "{} holds subspace {}, but the manifest says {subspace}", + self.describe(), + reader.subspace + )); + } + reader + } + } + } +} + +async fn restore_file(store: Store, blob_store: BlobStore, path: &Source) { + println!("Importing database dump from {}.", path.describe()); + + let mut reader = path.open(); let mut batch = BatchBuilder::new(); match reader.subspace { @@ -276,14 +462,20 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { struct KeyValueReader { subspace: u8, - file: FrameDecoder>, + file: FrameDecoder>>, } impl KeyValueReader { fn new(path: &Path) -> Self { - let mut file = FrameDecoder::new(BufReader::new( - File::open(path).failed("Failed to open file"), - )); + Self::from_reader( + Box::new(File::open(path).failed("Failed to open file")), + &path.display().to_string(), + ) + } + + // inbuxa: K-C11: any reader: a file, or a download from S3 + fn from_reader(reader: Box, path: &str) -> Self { + let mut file = FrameDecoder::new(BufReader::new(reader)); let mut buf = [0u8; 1]; file.read_exact(&mut buf) .failed(&format!("Failed to read magic marker from {path:?}")); diff --git a/crates/main/src/main.rs b/crates/main/src/main.rs index d4a05c6d..66f37fde 100644 --- a/crates/main/src/main.rs +++ b/crates/main/src/main.rs @@ -80,6 +80,12 @@ async fn main() -> std::io::Result<()> { std::process::exit(Box::pin(seed_and_exit(&server, &path, init.store_was_new)).await); } + // inbuxa: K-C11: --store-check runs a check and exits + if let Some(mode) = init.store_check.clone() { + let server = init.inner.build_server(); + std::process::exit(Box::pin(store_check_and_exit(&server, &mode)).await); + } + // Migrate database if let Err(err) = migration::try_migrate(&init.inner.build_server()).await { trc::event!( @@ -274,6 +280,60 @@ async fn seed_and_exit( flush_and_exit_code(code).await } +/// inbuxa: K-C11: `--store-check`: 0 found nothing wrong, 3 found problems, +/// 2 another check was running, 1 failed +async fn store_check_and_exit(server: &common::Server, mode: &str) -> i32 { + use services::inbuxa_store_check::{ + CheckMode as Mode, CheckRefusal as Refusal, CheckStatus as Status, + }; + + let mode = Mode::parse(mode).unwrap_or(Mode::Quick); + let (code, message) = match services::inbuxa_store_check::run_now(server, mode).await { + Ok(Ok(check)) => { + let p = &check.progress; + let summary = format!( + "Store check {} ({}): {} blobs, {} links, {} messages checked, {} bytes read. \ + Missing {}, damaged {}, unlinked {}, unreadable {}.", + check.id, + mode.as_str(), + p.blobs_checked, + p.links_checked, + p.messages_checked, + p.bytes_read, + p.missing, + p.damaged, + p.unlinked, + p.unreadable, + ); + match check.status { + Status::Done if p.problems() == 0 => (0, format!("{summary} Nothing wrong.")), + Status::Done => ( + 3, + format!( + "{summary} {} problems: the console's Store checks page lists them.", + p.problems() + ), + ), + status => ( + 1, + format!( + "{summary} It ended {}: {}", + status.as_str(), + check.error.as_deref().unwrap_or("no reason given") + ), + ), + } + } + Ok(Err(Refusal::Running(id))) => ( + 2, + format!("Store check {id} is running: one check runs at a time."), + ), + Err(err) => (1, format!("The store check failed: {err}")), + }; + println!("{message}"); + flush_and_exit_code(code).await +} + /// inbuxa: K-C9: `--upgrade-plan`: one line of JSON on stdout, for a /// script to read, and the exit code a shell can branch on async fn upgrade_plan_and_exit(server: &common::Server) -> i32 { diff --git a/crates/services/src/inbuxa_store_check.rs b/crates/services/src/inbuxa_store_check.rs index 5486d3c6..eca13d3c 100644 --- a/crates/services/src/inbuxa_store_check.rs +++ b/crates/services/src/inbuxa_store_check.rs @@ -74,6 +74,35 @@ pub async fn start( Ok(Ok(id)) } +// inbuxa: K-C11: for main's --store-check +pub use inbuxa_features::store_check::{ + Mode as CheckMode, Refusal as CheckRefusal, Status as CheckStatus, +}; + +/// inbuxa: K-C11: the node id a `--store-check` run records. Leased node +/// ids are 16-bit, so no server's start-up (SV-8) takes it for its own. +pub const ONE_SHOT_NODE_ID: u64 = 1 << 32; + +/// inbuxa: K-C11: `--store-check`: a check run here, to its end, for a +/// scheduled job. Saved like any other, so the console shows it. +pub async fn run_now(server: &Server, mode: Mode) -> trc::Result> { + let now = now(); + let check = StoreCheck::new(mode, "store-check job".to_string(), ONE_SHOT_NODE_ID, now); + let id = match model::create(server.store(), &check, now).await? { + Ok(id) => id, + Err(refusal) => return Ok(Err(refusal)), + }; + run(server, StoreCheck { id, ..check }).await; + Ok(model::get(server.store(), id) + .await? + .ok_or_else(|| { + trc::StoreEvent::NotFound + .into_err() + .details("The check went away") + }) + .map(Ok)?) +} + /// SV-8: asks a running check to stop. Any node can: the one running it /// sees the request at its next save. None if there's no such check. pub async fn stop(server: &Server, id: u32) -> trc::Result> { diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index 566e2a78..f509c565 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -65,7 +65,7 @@ foundation = ["foundationdb", "futures"] fdb-chunked-bm = [] # Blob stores -s3 = ["rust-s3", "dep:reqwest_s3"] +s3 = ["rust-s3", "dep:reqwest_s3", "futures"] azure = ["azure_core", "azure_storage", "azure_storage_blobs", "futures"] # In-memory stores diff --git a/crates/store/src/inbuxa_transfer.rs b/crates/store/src/inbuxa_transfer.rs new file mode 100644 index 00000000..4d5f1952 --- /dev/null +++ b/crates/store/src/inbuxa_transfer.rs @@ -0,0 +1,592 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +//! inbuxa: K-C11: an export written to, and read back from, an S3 bucket. +//! +//! `--export s3://bucket/prefix` streams each export file into a multipart +//! upload, so nothing the size of the store lands on local disk, and +//! `--import s3://bucket/prefix/` streams it back. The bucket's +//! address and keys come from the environment: +//! +//! - `INBUXA_EXPORT_S3_ENDPOINT`: an S3-compatible service (MinIO, Garage, +//! Ceph...), addressed path-style. Without it, AWS. +//! - `INBUXA_EXPORT_S3_REGION`: `us-east-1` unless given. +//! - `INBUXA_EXPORT_S3_ACCESS_KEY_FILE`, `INBUXA_EXPORT_S3_SECRET_KEY_FILE`: +//! files holding the keys. Without them, the usual AWS sources (the +//! `AWS_` variables, a profile, the instance's role). +//! +//! The export and import code is sync (a writer thread per file, readers in +//! the import tasks); these bridge it to the async S3 client. + +/// Whether a `--export`/`--import` value names a bucket. +pub fn is_s3_url(value: &str) -> bool { + value.starts_with("s3://") +} + +/// The name an export made now goes under: `20261119T204500Z`. +pub fn export_name(now: u64) -> String { + let days = (now / 86_400) as i64; + let secs = now % 86_400; + // Howard Hinnant's civil_from_days + let z = days + 719_468; + let era = z.div_euclid(146_097); + let doe = z.rem_euclid(146_097); + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let day = doy - (153 * mp + 2) / 5 + 1; + let month = if mp < 10 { mp + 3 } else { mp - 9 }; + let year = yoe + era * 400 + i64::from(month <= 2); + format!( + "{year:04}{month:02}{day:02}T{:02}{:02}{:02}Z", + secs / 3600, + (secs / 60) % 60, + secs % 60 + ) +} + +/// The size of upload part `n` (from 1): 16 MiB for the first thousand, +/// doubling every thousand after, up to 512 MiB. S3 takes at most 10,000 +/// parts, so one file can be nearly 3 TiB, and only one part is held at a +/// time. +pub fn part_size(part: u32) -> usize { + const MIB: usize = 1024 * 1024; + let tier = (part.saturating_sub(1) / 1000).min(5); + (16 * MIB) << tier +} + +/// The most parts S3 takes in one upload. +pub const MAX_PARTS: u32 = 10_000; + +#[cfg(feature = "s3")] +pub use imp::{PartReader, PartWriter, S3Transfer}; + +#[cfg(not(feature = "s3"))] +pub use none::{PartReader, PartWriter, S3Transfer}; + +#[cfg(feature = "s3")] +mod imp { + use super::{MAX_PARTS, part_size}; + use futures::StreamExt; + use s3::{Bucket, Region, creds::Credentials, serde_types::Part}; + use std::io::{Read, Write}; + use std::sync::mpsc as std_mpsc; + use tokio::sync::mpsc; + + const CONTENT_TYPE: &str = "application/octet-stream"; + const RETRIES: u32 = 5; + + #[derive(Clone)] + pub struct S3Transfer { + bucket: Box, + bucket_name: String, + /// No leading or trailing slash; may be empty. + prefix: String, + } + + fn env(name: &str) -> Option { + std::env::var(name) + .ok() + .map(|v| v.trim().to_string()) + .filter(|v| !v.is_empty()) + } + + fn key_file(name: &str) -> Result, String> { + match env(name) { + Some(path) => std::fs::read_to_string(&path) + .map(|v| Some(v.trim_end_matches(['\r', '\n']).to_string())) + .map_err(|err| format!("{name}: can't read {path}: {err}")), + None => Ok(None), + } + } + + impl S3Transfer { + /// `s3://bucket/prefix`, with the bucket's address and keys from the + /// environment. + pub fn open(url: &str) -> Result { + let rest = url + .strip_prefix("s3://") + .ok_or_else(|| format!("{url}: not an s3:// address"))?; + let (bucket_name, prefix) = rest.split_once('/').unwrap_or((rest, "")); + if bucket_name.is_empty() { + return Err(format!("{url}: no bucket named")); + } + let region_name = env("INBUXA_EXPORT_S3_REGION").unwrap_or_else(|| "us-east-1".into()); + let endpoint = env("INBUXA_EXPORT_S3_ENDPOINT"); + let region = match &endpoint { + Some(endpoint) => Region::Custom { + region: region_name, + endpoint: endpoint.trim_end_matches('/').to_string(), + }, + None => region_name + .parse() + .map_err(|err| format!("INBUXA_EXPORT_S3_REGION: {err}"))?, + }; + let access = key_file("INBUXA_EXPORT_S3_ACCESS_KEY_FILE")?; + let secret = key_file("INBUXA_EXPORT_S3_SECRET_KEY_FILE")?; + let credentials = match (&access, &secret) { + (Some(access), Some(secret)) => { + Credentials::new(Some(access), Some(secret), None, None, None) + } + (None, None) => Credentials::default(), + _ => { + return Err("INBUXA_EXPORT_S3_ACCESS_KEY_FILE and \ + INBUXA_EXPORT_S3_SECRET_KEY_FILE go together" + .into()); + } + } + .map_err(|err| format!("S3 credentials: {err}"))?; + let mut bucket = Bucket::new(bucket_name, region, credentials) + .map_err(|err| format!("S3 bucket {bucket_name}: {err}"))?; + if endpoint.is_some() { + bucket = bucket.with_path_style(); + } + Ok(S3Transfer { + bucket, + bucket_name: bucket_name.to_string(), + prefix: prefix.trim_matches('/').to_string(), + }) + } + + /// The same bucket, one level down. + pub fn join(&self, name: &str) -> Self { + S3Transfer { + prefix: self.key(name), + ..self.clone() + } + } + + pub fn location(&self) -> String { + format!("s3://{}/{}", self.bucket_name, self.prefix) + } + + fn key(&self, name: &str) -> String { + if self.prefix.is_empty() { + name.to_string() + } else { + format!("{}/{name}", self.prefix) + } + } + + /// A small object, whole. + pub async fn put(&self, name: &str, data: &[u8]) -> Result<(), String> { + let key = self.key(name); + let mut last = String::new(); + for attempt in 0..RETRIES { + match self.bucket.put_object(&key, data).await { + Ok(r) if (200..300).contains(&r.status_code()) => return Ok(()), + Ok(r) => { + last = format!( + "{}: {}", + r.status_code(), + String::from_utf8_lossy(r.as_slice()) + ) + } + Err(err) => last = err.to_string(), + } + wait(attempt).await; + } + Err(format!( + "Failed to write {}: {last}", + self.location_of(name) + )) + } + + /// A small object, whole; None when it isn't there. + pub async fn get(&self, name: &str) -> Result>, String> { + let key = self.key(name); + let mut last = String::new(); + for attempt in 0..RETRIES { + match self.bucket.get_object(&key).await { + Ok(r) if (200..300).contains(&r.status_code()) => { + return Ok(Some(r.to_vec())); + } + Ok(r) if r.status_code() == 404 => return Ok(None), + Ok(r) => last = format!("{}", r.status_code()), + Err(s3::error::S3Error::HttpFailWithBody(404, _)) => return Ok(None), + Err(err) => last = err.to_string(), + } + wait(attempt).await; + } + Err(format!("Failed to read {}: {last}", self.location_of(name))) + } + + /// The names one level down that hold `marker`: the exports under a + /// prefix. + pub async fn children_with(&self, marker: &str) -> Result, String> { + let prefix = if self.prefix.is_empty() { + String::new() + } else { + format!("{}/", self.prefix) + }; + let results = self + .bucket + .list(prefix.clone(), None) + .await + .map_err(|err| format!("Failed to list {}: {err}", self.location()))?; + let mut names = results + .iter() + .flat_map(|r| r.contents.iter()) + .filter_map(|o| o.key.strip_prefix(&prefix)) + .filter_map(|rest| rest.strip_suffix(marker)?.strip_suffix('/')) + .filter(|name| !name.is_empty() && !name.contains('/')) + .map(str::to_string) + .collect::>(); + names.sort(); + Ok(names) + } + + fn location_of(&self, name: &str) -> String { + format!("s3://{}/{}", self.bucket_name, self.key(name)) + } + + /// A writer whose bytes go up as a multipart upload under `name`, + /// for a thread outside the runtime; the upload runs on the + /// runtime. Dropping the writer sends the last part; the handle + /// ends when the upload is complete. + pub fn writer( + &self, + name: &str, + ) -> (PartWriter, tokio::task::JoinHandle>) { + let (tx, rx) = mpsc::channel::>(1); + let this = self.clone(); + let name = name.to_string(); + let handle = tokio::spawn(async move { this.upload(&name, rx).await }); + ( + PartWriter { + tx: Some(tx), + buf: Vec::with_capacity(part_size(1)), + part: 1, + }, + handle, + ) + } + + async fn upload(&self, name: &str, mut rx: mpsc::Receiver>) -> Result { + let key = self.key(name); + let location = self.location_of(name); + let upload_id = self + .bucket + .initiate_multipart_upload(&key, CONTENT_TYPE) + .await + .map_err(|err| format!("Failed to start uploading {location}: {err}"))? + .upload_id; + let mut parts: Vec = Vec::new(); + let mut total = 0u64; + let result: Result<(), String> = async { + while let Some(chunk) = rx.recv().await { + let number = parts.len() as u32 + 1; + if number > MAX_PARTS { + return Err(format!("{location}: too large for one upload")); + } + total += chunk.len() as u64; + parts.push( + self.put_part(chunk, &key, number, &upload_id, &location) + .await?, + ); + } + Ok(()) + } + .await; + if let Err(err) = result { + // Dropping rx fails the writer's next send, which stops it + drop(rx); + let _ = self.bucket.abort_upload(&key, &upload_id).await; + return Err(err); + } + let mut last = String::new(); + for attempt in 0..RETRIES { + match self + .bucket + .complete_multipart_upload(&key, &upload_id, parts.clone()) + .await + { + Ok(r) if (200..300).contains(&r.status_code()) => return Ok(total), + Ok(r) => last = format!("{}", r.status_code()), + Err(err) => last = err.to_string(), + } + wait(attempt).await; + } + let _ = self.bucket.abort_upload(&key, &upload_id).await; + Err(format!("Failed to finish uploading {location}: {last}")) + } + + async fn put_part( + &self, + chunk: Vec, + key: &str, + number: u32, + upload_id: &str, + location: &str, + ) -> Result { + let mut last = String::new(); + for attempt in 0..RETRIES { + match self + .bucket + .put_multipart_chunk(chunk.clone(), key, number, upload_id, CONTENT_TYPE) + .await + { + Ok(part) => return Ok(part), + Err(err) => last = err.to_string(), + } + wait(attempt).await; + } + Err(format!( + "Failed to upload part {number} of {location}: {last}" + )) + } + + /// A reader of the object `name`, for code that reads synchronously + /// on the runtime's workers: the download is a task on the runtime, + /// and a reader that has to wait for it hands its worker over. + pub fn reader(&self, name: &str) -> PartReader { + let (tx, rx) = std_mpsc::sync_channel::, String>>(4); + let this = self.clone(); + let key = self.key(name); + let location = self.location_of(name); + tokio::spawn(async move { + { + let mut stream = match this.bucket.get_object_stream(&key).await { + Ok(stream) if (200..300).contains(&stream.status_code) => stream, + Ok(stream) => { + let _ = send_chunk( + &tx, + Err(format!("Failed to read {location}: {}", stream.status_code)), + ); + return; + } + Err(err) => { + let _ = + send_chunk(&tx, Err(format!("Failed to read {location}: {err}"))); + return; + } + }; + while let Some(chunk) = stream.bytes().next().await { + let sent = match chunk { + Ok(bytes) => send_chunk(&tx, Ok(bytes.to_vec())), + Err(err) => { + let _ = send_chunk( + &tx, + Err(format!("Failed to read {location}: {err}")), + ); + return; + } + }; + if sent.is_err() { + return; + } + } + } + }); + PartReader { + rx, + buf: Vec::new(), + pos: 0, + } + } + } + + /// Whether this thread is a worker of a multi-threaded runtime, where a + /// wait can hand the worker over (`block_in_place`) + fn on_worker() -> bool { + tokio::runtime::Handle::try_current() + .is_ok_and(|h| h.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread) + } + + /// A send that, when it has to wait for the reader, hands the worker over + fn send_chunk( + tx: &std_mpsc::SyncSender, String>>, + item: Result, String>, + ) -> Result<(), ()> { + match tx.try_send(item) { + Ok(()) => Ok(()), + Err(std_mpsc::TrySendError::Disconnected(_)) => Err(()), + Err(std_mpsc::TrySendError::Full(item)) if on_worker() => { + tokio::task::block_in_place(|| tx.send(item)).map_err(|_| ()) + } + Err(std_mpsc::TrySendError::Full(item)) => tx.send(item).map_err(|_| ()), + } + } + + async fn wait(attempt: u32) { + tokio::time::sleep(std::time::Duration::from_millis(250 << attempt.min(5))).await; + } + + /// See [`S3Transfer::writer`]. + pub struct PartWriter { + tx: Option>>, + buf: Vec, + part: u32, + } + + impl PartWriter { + fn send(&mut self) -> std::io::Result<()> { + let next = part_size(self.part + 1); + let chunk = std::mem::replace(&mut self.buf, Vec::with_capacity(next)); + self.part += 1; + match &self.tx { + Some(tx) => tx.blocking_send(chunk).map_err(|_| { + std::io::Error::other("the upload stopped: its error is on the next line") + }), + None => Err(std::io::Error::other("the upload is finished")), + } + } + } + + impl Write for PartWriter { + fn write(&mut self, data: &[u8]) -> std::io::Result { + let room = part_size(self.part) - self.buf.len(); + let take = room.min(data.len()); + self.buf.extend_from_slice(&data[..take]); + if self.buf.len() == part_size(self.part) { + self.send()?; + } + Ok(take) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + impl Drop for PartWriter { + fn drop(&mut self) { + // The last part, which may be short; even an empty file is one + // part + if !self.buf.is_empty() || self.part == 1 { + let _ = self.send(); + } + self.tx = None; + } + } + + /// See [`S3Transfer::reader`]. + pub struct PartReader { + rx: std_mpsc::Receiver, String>>, + buf: Vec, + pos: usize, + } + + impl Read for PartReader { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + while self.pos == self.buf.len() { + // Readers run on the runtime's workers (the import's tasks), + // and the download needs one to make progress: a wait hands + // this worker over first, or a pod with one CPU's worth of + // workers would stop here + let next = match self.rx.try_recv() { + Ok(next) => Ok(next), + Err(std_mpsc::TryRecvError::Disconnected) => Err(std_mpsc::RecvError), + Err(std_mpsc::TryRecvError::Empty) => { + if on_worker() { + tokio::task::block_in_place(|| self.rx.recv()) + } else { + self.rx.recv() + } + } + }; + match next { + Ok(Ok(chunk)) => { + self.buf = chunk; + self.pos = 0; + } + Ok(Err(err)) => return Err(std::io::Error::other(err)), + // The download ended + Err(_) => return Ok(0), + } + } + let n = out.len().min(self.buf.len() - self.pos); + out[..n].copy_from_slice(&self.buf[self.pos..self.pos + n]); + self.pos += n; + Ok(n) + } + } +} + +#[cfg(not(feature = "s3"))] +mod none { + //! This build has no S3 client: [`S3Transfer::open`] always refuses, so + //! nothing else here is ever reached. + + #[derive(Clone)] + pub struct S3Transfer; + pub struct PartWriter; + pub struct PartReader; + + impl S3Transfer { + pub fn open(_url: &str) -> Result { + Err("this build has no S3 support (the s3 feature)".into()) + } + pub fn join(&self, _name: &str) -> Self { + unreachable!() + } + pub fn location(&self) -> String { + unreachable!() + } + pub async fn put(&self, _name: &str, _data: &[u8]) -> Result<(), String> { + unreachable!() + } + pub async fn get(&self, _name: &str) -> Result>, String> { + unreachable!() + } + pub async fn children_with(&self, _marker: &str) -> Result, String> { + unreachable!() + } + pub fn writer( + &self, + _name: &str, + ) -> (PartWriter, tokio::task::JoinHandle>) { + unreachable!() + } + pub fn reader(&self, _name: &str) -> PartReader { + unreachable!() + } + } + + impl std::io::Write for PartWriter { + fn write(&mut self, _data: &[u8]) -> std::io::Result { + unreachable!() + } + fn flush(&mut self) -> std::io::Result<()> { + unreachable!() + } + } + + impl std::io::Read for PartReader { + fn read(&mut self, _out: &mut [u8]) -> std::io::Result { + unreachable!() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn export_names_sort_by_time() { + assert_eq!(export_name(0), "19700101T000000Z"); + assert_eq!(export_name(1_795_552_496), "20261124T203456Z"); + assert_eq!(export_name(951_782_400), "20000229T000000Z"); + assert!(export_name(1_795_552_496) < export_name(1_795_552_497)); + } + + #[test] + fn parts_grow_to_nearly_three_tebibytes() { + assert_eq!(part_size(1), 16 << 20); + assert_eq!(part_size(1000), 16 << 20); + assert_eq!(part_size(1001), 32 << 20); + assert_eq!(part_size(MAX_PARTS), 512 << 20); + let total: u64 = (1..=MAX_PARTS).map(|n| part_size(n) as u64).sum(); + assert!(total > (29u64 << 40) / 10, "{total}"); + } + + #[test] + fn s3_urls() { + assert!(is_s3_url("s3://backups/inbuxa")); + assert!(!is_s3_url("/var/backups")); + } +} diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 440c0595..a7b69d2b 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -11,6 +11,8 @@ pub mod backend; pub mod build; pub mod dispatch; +// inbuxa: K-C11: exports to and from S3 +pub mod inbuxa_transfer; pub mod query; pub mod registry; pub mod search; diff --git a/tests/src/store/import_export.rs b/tests/src/store/import_export.rs index 20d72aa3..f94d23a7 100644 --- a/tests/src/store/import_export.rs +++ b/tests/src/store/import_export.rs @@ -278,6 +278,31 @@ pub async fn test(test: &TestServer) { .backup(BackupParams::new(temp_dir.path.clone())) .await; + // inbuxa: K-C11: a record in the fork's subspace, which every start + // writes (here whichever one the test server wrote first) + let mut startup_key = None; + db.iterate( + IterateParams::new( + AnyKey { + subspace: SUBSPACE_INBUXA, + key: vec![0u8], + }, + AnyKey { + subspace: SUBSPACE_INBUXA, + key: vec![u8::MAX; 32], + }, + ) + .no_values(), + |key, _| { + startup_key = Some(key.to_vec()); + Ok(false) + }, + ) + .await + .unwrap(); + let startup_key = + startup_key.expect("the test server's start wrote nothing to the fork's subspace"); + // Destroy store println!("Destroying store..."); store_destroy(&db).await; @@ -293,6 +318,16 @@ pub async fn test(test: &TestServer) { .write("localhost") .finalize(), ); + // inbuxa: K-C11: and over what any start writes first, the import's own + // too: a record in the fork's subspace, with a value the import has to + // set over. Refused as data in the store until rc.34. + batch.set( + ValueClass::Any(AnyClass { + subspace: SUBSPACE_INBUXA, + key: startup_key, + }), + b"written at start-up".to_vec(), + ); db.write(batch.build_all()).await.unwrap(); for (key, _) in &named_blobs { test.server.blob_store().delete_blob(key).await.unwrap(); -- 2.54.0