From 3958f5f17e177137a80aed19ed352e3ecdbd3051 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Fri, 20 Nov 2026 18:21:09 -0700 Subject: [PATCH] Import: run on a fresh store, and put blobs in the imported blob store Backported from 27.1 (#491), the two import fixes only; the S3 export and store-check command stay on 27.1. Fixed - An import into a fresh store refused to run: every start writes the fork's subspace (where the blob store is, permission grants, the log files' setting), the import's own start too, and the import took that for data. Since the blob store change (#333): 2026.11.9.1. The fork's subspace is no longer counted; its records are keyed values an import sets over. The store test now imports over a start-up record, and fails without this. - An import wrote the blobs into the data store, the empty store's blob store, even when the imported settings name another (S3, say): the restored server then found none of them. Blobs are imported last, into the blob store the imported x:BlobStore names. Tested: store tests pass on SQLite and RocksDB; without the first fix the store test exits at the import's refusal. --- crates/common/src/manager/restore.rs | 68 ++++++++++++++++++++++++++-- tests/src/store/import_export.rs | 35 ++++++++++++++ 2 files changed, 100 insertions(+), 3 deletions(-) diff --git a/crates/common/src/manager/restore.rs b/crates/common/src/manager/restore.rs index 99b37800..bd79f3c3 100644 --- a/crates/common/src/manager/restore.rs +++ b/crates/common/src/manager/restore.rs @@ -22,7 +22,7 @@ use std::{ path::{Path, PathBuf}, }; use store::{ - BlobStore, IterateParams, SUBSPACE_BLOBS, SUBSPACE_COUNTER, SUBSPACE_INDEXES, + BlobStore, IterateParams, SUBSPACE_BLOBS, SUBSPACE_COUNTER, SUBSPACE_INBUXA, SUBSPACE_INDEXES, SUBSPACE_PROPERTY, SUBSPACE_QUOTA, SUBSPACE_REGISTRY_PK, SUBSPACE_TELEMETRY_SPAN, Store, U32_LEN, write::{ @@ -55,6 +55,14 @@ impl Core { let mut conflicts = Vec::new(); for path in &paths { let subspace = KeyValueReader::new(path).subspace; + // inbuxa: every start writes the fork's own subspace, this + // import's too: where the blob store is (W), the permission + // grants (P), the log files' setting (T). Its records are keyed + // values, which an import sets over, never duplicates; a store in + // use still has data in the other subspaces, which are checked. + if subspace == SUBSPACE_INBUXA { + continue; + } if subspace_has_data(&self.storage.data, subspace).await { conflicts.push(path.clone()); } @@ -80,22 +88,76 @@ impl Core { imported.sort_unstable(); imported.dedup(); + // inbuxa: the blobs last, into the blob store the imported settings + // name. This process started on an empty store, whose blob store is + // the data store; a server started on the imported one reads blobs + // from wherever its x:BlobStore says (S3, say), so blobs written to + // the data store would be missing there. + let (blobs, rest): (Vec<_>, Vec<_>) = paths + .into_iter() + .partition(|path| KeyValueReader::new(path).subspace == SUBSPACE_BLOBS); + let mut tasks = Vec::new(); - for path in paths { + for path in rest { let storage = self.storage.clone(); let blob_store = self.storage.blob.clone(); tasks.push(tokio::spawn(async move { restore_file(storage.data, blob_store, &path).await; })); } - for task in tasks { task.await.failed("Failed to wait for task"); } + if !blobs.is_empty() { + let blob_store = self.imported_blob_store().await; + let mut tasks = Vec::new(); + for path in blobs { + let storage = self.storage.clone(); + let blob_store = blob_store.clone(); + tasks.push(tokio::spawn(async move { + restore_file(storage.data, blob_store, &path).await; + })); + } + for task in tasks { + task.await.failed("Failed to wait for task"); + } + } + imported } + /// inbuxa: the blob store the store's x:BlobStore names now, after the + /// import wrote it + async fn imported_blob_store(&self) -> BlobStore { + use registry::{schema::structs::BlobStore as BlobStoreSetting, types::ObjectImpl}; + let setting = self + .storage + .registry + .object::(BlobStoreSetting::OBJECT.singleton().id()) + .await + .failed("Failed to read the imported blob store setting") + .unwrap_or_default(); + if matches!(setting, BlobStoreSetting::Default) { + println!("Importing blobs into the data store, as the export's settings say."); + return self.storage.data.clone().into(); + } + let mut warnings = Vec::new(); + let store = BlobStore::open(setting, &self.storage.data, &mut warnings) + .await + .unwrap_or_else(|err| { + failed(&format!( + "Can't open the blob store the export's settings name, to import the \ + blobs into: {err}" + )) + }); + for warning in warnings { + eprintln!("Blob store: {warning}"); + } + println!("Importing blobs into the blob store the export's settings name."); + store + } + /// inbuxa: an export never carries the full-text index. It is built by /// and for one search backend (the SQL stores index into their own /// tables, the key-value stores into a subspace, external engines keep it diff --git a/tests/src/store/import_export.rs b/tests/src/store/import_export.rs index 3e95472e..11b87d78 100644 --- a/tests/src/store/import_export.rs +++ b/tests/src/store/import_export.rs @@ -259,6 +259,31 @@ pub async fn test(test: &TestServer) { .backup(BackupParams::new(temp_dir.path.clone())) .await; + // inbuxa: K-C11: a record in the fork's subspace, which every start + // writes (here whichever one the test server wrote first) + let mut startup_key = None; + db.iterate( + IterateParams::new( + AnyKey { + subspace: SUBSPACE_INBUXA, + key: vec![0u8], + }, + AnyKey { + subspace: SUBSPACE_INBUXA, + key: vec![u8::MAX; 32], + }, + ) + .no_values(), + |key, _| { + startup_key = Some(key.to_vec()); + Ok(false) + }, + ) + .await + .unwrap(); + let startup_key = + startup_key.expect("the test server's start wrote nothing to the fork's subspace"); + // Destroy store println!("Destroying store..."); store_destroy(&db).await; @@ -274,6 +299,16 @@ pub async fn test(test: &TestServer) { .write("localhost") .finalize(), ); + // inbuxa: K-C11: and over what any start writes first, the import's own + // too: a record in the fork's subspace, with a value the import has to + // set over. Refused as data in the store until rc.34. + batch.set( + ValueClass::Any(AnyClass { + subspace: SUBSPACE_INBUXA, + key: startup_key, + }), + b"written at start-up".to_vec(), + ); db.write(batch.build_all()).await.unwrap(); for (key, _) in &named_blobs { test.server.blob_store().delete_blob(key).await.unwrap(); -- 2.54.0