Import: run on a fresh store, and put blobs in the imported blob store (2026.x backport of #491's fixes) #520

Merged
jcoffey-dev merged 1 commits from fix/import-fresh-store-2026 into release/2026 2026-11-21 01:30:39 +00:00
2 changed files with 100 additions and 3 deletions

No files matched your search

+65 -3
View File
@@ -22,7 +22,7 @@ use std::{
path::{Path, PathBuf},
};
use store::{
BlobStore, IterateParams, SUBSPACE_BLOBS, SUBSPACE_COUNTER, SUBSPACE_INDEXES,
BlobStore, IterateParams, SUBSPACE_BLOBS, SUBSPACE_COUNTER, SUBSPACE_INBUXA, SUBSPACE_INDEXES,
SUBSPACE_PROPERTY, SUBSPACE_QUOTA, SUBSPACE_REGISTRY_PK, SUBSPACE_TELEMETRY_SPAN, Store,
U32_LEN,
write::{
@@ -55,6 +55,14 @@ impl Core {
let mut conflicts = Vec::new();
for path in &paths {
let subspace = KeyValueReader::new(path).subspace;
// inbuxa: every start writes the fork's own subspace, this
// import's too: where the blob store is (W), the permission
// grants (P), the log files' setting (T). Its records are keyed
// values, which an import sets over, never duplicates; a store in
// use still has data in the other subspaces, which are checked.
if subspace == SUBSPACE_INBUXA {
continue;
}
if subspace_has_data(&self.storage.data, subspace).await {
conflicts.push(path.clone());
}
@@ -80,22 +88,76 @@ impl Core {
imported.sort_unstable();
imported.dedup();
// inbuxa: the blobs last, into the blob store the imported settings
// name. This process started on an empty store, whose blob store is
// the data store; a server started on the imported one reads blobs
// from wherever its x:BlobStore says (S3, say), so blobs written to
// the data store would be missing there.
let (blobs, rest): (Vec<_>, Vec<_>) = paths
.into_iter()
.partition(|path| KeyValueReader::new(path).subspace == SUBSPACE_BLOBS);
let mut tasks = Vec::new();
for path in paths {
for path in rest {
let storage = self.storage.clone();
let blob_store = self.storage.blob.clone();
tasks.push(tokio::spawn(async move {
restore_file(storage.data, blob_store, &path).await;
}));
}
for task in tasks {
task.await.failed("Failed to wait for task");
}
if !blobs.is_empty() {
let blob_store = self.imported_blob_store().await;
let mut tasks = Vec::new();
for path in blobs {
let storage = self.storage.clone();
let blob_store = blob_store.clone();
tasks.push(tokio::spawn(async move {
restore_file(storage.data, blob_store, &path).await;
}));
}
for task in tasks {
task.await.failed("Failed to wait for task");
}
}
imported
}
/// inbuxa: the blob store the store's x:BlobStore names now, after the
/// import wrote it
async fn imported_blob_store(&self) -> BlobStore {
use registry::{schema::structs::BlobStore as BlobStoreSetting, types::ObjectImpl};
let setting = self
.storage
.registry
.object::<BlobStoreSetting>(BlobStoreSetting::OBJECT.singleton().id())
.await
.failed("Failed to read the imported blob store setting")
.unwrap_or_default();
if matches!(setting, BlobStoreSetting::Default) {
println!("Importing blobs into the data store, as the export's settings say.");
return self.storage.data.clone().into();
}
let mut warnings = Vec::new();
let store = BlobStore::open(setting, &self.storage.data, &mut warnings)
.await
.unwrap_or_else(|err| {
failed(&format!(
"Can't open the blob store the export's settings name, to import the \
blobs into: {err}"
))
});
for warning in warnings {
eprintln!("Blob store: {warning}");
}
println!("Importing blobs into the blob store the export's settings name.");
store
}
/// inbuxa: an export never carries the full-text index. It is built by
/// and for one search backend (the SQL stores index into their own
/// tables, the key-value stores into a subspace, external engines keep it
+35
View File
@@ -259,6 +259,31 @@ pub async fn test(test: &TestServer) {
.backup(BackupParams::new(temp_dir.path.clone()))
.await;
// inbuxa: K-C11: a record in the fork's subspace, which every start
// writes (here whichever one the test server wrote first)
let mut startup_key = None;
db.iterate(
IterateParams::new(
AnyKey {
subspace: SUBSPACE_INBUXA,
key: vec![0u8],
},
AnyKey {
subspace: SUBSPACE_INBUXA,
key: vec![u8::MAX; 32],
},
)
.no_values(),
|key, _| {
startup_key = Some(key.to_vec());
Ok(false)
},
)
.await
.unwrap();
let startup_key =
startup_key.expect("the test server's start wrote nothing to the fork's subspace");
// Destroy store
println!("Destroying store...");
store_destroy(&db).await;
@@ -274,6 +299,16 @@ pub async fn test(test: &TestServer) {
.write("localhost")
.finalize(),
);
// inbuxa: K-C11: and over what any start writes first, the import's own
// too: a record in the fork's subspace, with a value the import has to
// set over. Refused as data in the store until rc.34.
batch.set(
ValueClass::Any(AnyClass {
subspace: SUBSPACE_INBUXA,
key: startup_key,
}),
b"written at start-up".to_vec(),
);
db.write(batch.build_all()).await.unwrap();
for (key, _) in &named_blobs {
test.server.blob_store().delete_blob(key).await.unwrap();