Import: run on a fresh store, and put blobs in the imported blob store (2026.x backport of #491's fixes) #520
No files matched your search
@@ -22,7 +22,7 @@ use std::{
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
use store::{
|
||||
BlobStore, IterateParams, SUBSPACE_BLOBS, SUBSPACE_COUNTER, SUBSPACE_INDEXES,
|
||||
BlobStore, IterateParams, SUBSPACE_BLOBS, SUBSPACE_COUNTER, SUBSPACE_INBUXA, SUBSPACE_INDEXES,
|
||||
SUBSPACE_PROPERTY, SUBSPACE_QUOTA, SUBSPACE_REGISTRY_PK, SUBSPACE_TELEMETRY_SPAN, Store,
|
||||
U32_LEN,
|
||||
write::{
|
||||
@@ -55,6 +55,14 @@ impl Core {
|
||||
let mut conflicts = Vec::new();
|
||||
for path in &paths {
|
||||
let subspace = KeyValueReader::new(path).subspace;
|
||||
// inbuxa: every start writes the fork's own subspace, this
|
||||
// import's too: where the blob store is (W), the permission
|
||||
// grants (P), the log files' setting (T). Its records are keyed
|
||||
// values, which an import sets over, never duplicates; a store in
|
||||
// use still has data in the other subspaces, which are checked.
|
||||
if subspace == SUBSPACE_INBUXA {
|
||||
continue;
|
||||
}
|
||||
if subspace_has_data(&self.storage.data, subspace).await {
|
||||
conflicts.push(path.clone());
|
||||
}
|
||||
@@ -80,22 +88,76 @@ impl Core {
|
||||
imported.sort_unstable();
|
||||
imported.dedup();
|
||||
|
||||
// inbuxa: the blobs last, into the blob store the imported settings
|
||||
// name. This process started on an empty store, whose blob store is
|
||||
// the data store; a server started on the imported one reads blobs
|
||||
// from wherever its x:BlobStore says (S3, say), so blobs written to
|
||||
// the data store would be missing there.
|
||||
let (blobs, rest): (Vec<_>, Vec<_>) = paths
|
||||
.into_iter()
|
||||
.partition(|path| KeyValueReader::new(path).subspace == SUBSPACE_BLOBS);
|
||||
|
||||
let mut tasks = Vec::new();
|
||||
for path in paths {
|
||||
for path in rest {
|
||||
let storage = self.storage.clone();
|
||||
let blob_store = self.storage.blob.clone();
|
||||
tasks.push(tokio::spawn(async move {
|
||||
restore_file(storage.data, blob_store, &path).await;
|
||||
}));
|
||||
}
|
||||
|
||||
for task in tasks {
|
||||
task.await.failed("Failed to wait for task");
|
||||
}
|
||||
|
||||
if !blobs.is_empty() {
|
||||
let blob_store = self.imported_blob_store().await;
|
||||
let mut tasks = Vec::new();
|
||||
for path in blobs {
|
||||
let storage = self.storage.clone();
|
||||
let blob_store = blob_store.clone();
|
||||
tasks.push(tokio::spawn(async move {
|
||||
restore_file(storage.data, blob_store, &path).await;
|
||||
}));
|
||||
}
|
||||
for task in tasks {
|
||||
task.await.failed("Failed to wait for task");
|
||||
}
|
||||
}
|
||||
|
||||
imported
|
||||
}
|
||||
|
||||
/// inbuxa: the blob store the store's x:BlobStore names now, after the
|
||||
/// import wrote it
|
||||
async fn imported_blob_store(&self) -> BlobStore {
|
||||
use registry::{schema::structs::BlobStore as BlobStoreSetting, types::ObjectImpl};
|
||||
let setting = self
|
||||
.storage
|
||||
.registry
|
||||
.object::<BlobStoreSetting>(BlobStoreSetting::OBJECT.singleton().id())
|
||||
.await
|
||||
.failed("Failed to read the imported blob store setting")
|
||||
.unwrap_or_default();
|
||||
if matches!(setting, BlobStoreSetting::Default) {
|
||||
println!("Importing blobs into the data store, as the export's settings say.");
|
||||
return self.storage.data.clone().into();
|
||||
}
|
||||
let mut warnings = Vec::new();
|
||||
let store = BlobStore::open(setting, &self.storage.data, &mut warnings)
|
||||
.await
|
||||
.unwrap_or_else(|err| {
|
||||
failed(&format!(
|
||||
"Can't open the blob store the export's settings name, to import the \
|
||||
blobs into: {err}"
|
||||
))
|
||||
});
|
||||
for warning in warnings {
|
||||
eprintln!("Blob store: {warning}");
|
||||
}
|
||||
println!("Importing blobs into the blob store the export's settings name.");
|
||||
store
|
||||
}
|
||||
|
||||
/// inbuxa: an export never carries the full-text index. It is built by
|
||||
/// and for one search backend (the SQL stores index into their own
|
||||
/// tables, the key-value stores into a subspace, external engines keep it
|
||||
|
||||
@@ -259,6 +259,31 @@ pub async fn test(test: &TestServer) {
|
||||
.backup(BackupParams::new(temp_dir.path.clone()))
|
||||
.await;
|
||||
|
||||
// inbuxa: K-C11: a record in the fork's subspace, which every start
|
||||
// writes (here whichever one the test server wrote first)
|
||||
let mut startup_key = None;
|
||||
db.iterate(
|
||||
IterateParams::new(
|
||||
AnyKey {
|
||||
subspace: SUBSPACE_INBUXA,
|
||||
key: vec![0u8],
|
||||
},
|
||||
AnyKey {
|
||||
subspace: SUBSPACE_INBUXA,
|
||||
key: vec![u8::MAX; 32],
|
||||
},
|
||||
)
|
||||
.no_values(),
|
||||
|key, _| {
|
||||
startup_key = Some(key.to_vec());
|
||||
Ok(false)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let startup_key =
|
||||
startup_key.expect("the test server's start wrote nothing to the fork's subspace");
|
||||
|
||||
// Destroy store
|
||||
println!("Destroying store...");
|
||||
store_destroy(&db).await;
|
||||
@@ -274,6 +299,16 @@ pub async fn test(test: &TestServer) {
|
||||
.write("localhost")
|
||||
.finalize(),
|
||||
);
|
||||
// inbuxa: K-C11: and over what any start writes first, the import's own
|
||||
// too: a record in the fork's subspace, with a value the import has to
|
||||
// set over. Refused as data in the store until rc.34.
|
||||
batch.set(
|
||||
ValueClass::Any(AnyClass {
|
||||
subspace: SUBSPACE_INBUXA,
|
||||
key: startup_key,
|
||||
}),
|
||||
b"written at start-up".to_vec(),
|
||||
);
|
||||
db.write(batch.build_all()).await.unwrap();
|
||||
for (key, _) in &named_blobs {
|
||||
test.server.blob_store().delete_blob(key).await.unwrap();
|
||||
|
||||
Reference in new issue
Block a user