Backups for Kubernetes: export to and import from S3, store checks as a command (K-C11) #491

Merged
jcoffey-dev merged 1 commits from feat/k8s-backups into main 2026-11-19 21:04:02 +00:00
11 changed files with 1112 additions and 59 deletions

No files matched your search

+1 -1
View File
@@ -11,7 +11,7 @@ jobs:
announce:
runs-on: light
steps:
- uses: coffey-labs/actions/discourse-release@ba4cde7c3b47cc9f93c0080da8ba838a1ef1845b
- uses: coffey-labs/actions/discourse-release@958b3a35a345744abd236ca5abf679cbedaca7a0
with:
api-key: ${{ secrets.DISCOURSE_RELEASE_KEY }}
discord-webhook: ${{ secrets.DISCORD_RELEASE_WEBHOOK }}
+1 -1
View File
@@ -511,7 +511,7 @@ jobs:
if: ${{ always() && ((needs.release.result == 'success' && needs.binaries.result == 'success') || needs.github.result == 'success') }}
runs-on: light
steps:
- uses: coffey-labs/actions/discourse-release@ba4cde7c3b47cc9f93c0080da8ba838a1ef1845b
- uses: coffey-labs/actions/discourse-release@958b3a35a345744abd236ca5abf679cbedaca7a0
with:
api-key: ${{ secrets.DISCOURSE_RELEASE_KEY }}
discord-webhook: ${{ secrets.DISCORD_RELEASE_WEBHOOK }}
+138 -31
View File
@@ -11,7 +11,7 @@ use ahash::AHashSet;
use lz4_flex::frame::FrameEncoder;
use std::{
io::{BufWriter, Write},
path::{Path, PathBuf},
path::PathBuf,
sync::mpsc::{self, SyncSender},
};
use store::{
@@ -19,7 +19,7 @@ use store::{
*,
};
use types::blob_hash::{BLOB_HASH_LEN, BlobHash};
use utils::{UnwrapFailure, codec::leb128::Leb128_};
use utils::{UnwrapFailure, codec::leb128::Leb128_, failed};
pub(super) const MAGIC_MARKER: u8 = 123;
@@ -52,9 +52,31 @@ pub struct BackupParams {
families: AHashSet<Family>,
}
/// inbuxa: K-C11: where the export files go
enum Sink {
Dir(PathBuf),
S3(store::inbuxa_transfer::S3Transfer),
}
type Upload = tokio::task::JoinHandle<Result<u64, String>>;
/// inbuxa: K-C11: an export in S3 ends with this, written last, naming its
/// files: an export without one didn't finish.
pub const S3_EXPORT_MANIFEST: &str = "export.json";
impl Core {
pub async fn backup(&self, mut params: BackupParams) {
if !params.dest.exists() {
// inbuxa: K-C11: s3://bucket/prefix exports to a new folder there
let dest_text = params.dest.to_string_lossy().to_string();
let sink = if store::inbuxa_transfer::is_s3_url(&dest_text) {
let transfer = store::inbuxa_transfer::S3Transfer::open(&dest_text)
.unwrap_or_else(|err| failed(&format!("Export failed: {err}")));
Sink::S3(transfer.join(&store::inbuxa_transfer::export_name(store::write::now())))
} else {
Sink::Dir(params.dest.clone())
};
if let Sink::S3(_) = &sink {
} else if !params.dest.exists() {
std::fs::create_dir_all(&params.dest).failed("Failed to create backup directory");
} else if !params.dest.is_dir() {
eprintln!("Backup destination {:?} is not a directory.", params.dest);
@@ -73,7 +95,9 @@ impl Core {
.failed("Could not retrieve database schema version.")
.failed("Could not retrieve database schema version.");
let mut families_given = true;
if params.families.is_empty() {
families_given = false;
params.families = [
Family::Data,
Family::Registry,
@@ -89,34 +113,81 @@ impl Core {
.collect();
}
for subspace in params
.families
.into_iter()
.flat_map(|f| f.subspaces())
.copied()
{
// inbuxa: K-C11: blobs last. An export taken while the server runs
// then holds every blob the rest of it names: a message stored
// during the export is either in it whole or not at all. (In a set's
// order, the blobs could go first, and the messages stored after
// them would point at blobs the export never got.)
let mut families = params.families.into_iter().collect::<Vec<_>>();
families.sort_by_key(|f| (*f == Family::Blob, *f as u8));
let mut uploads: Vec<(String, u8, Upload)> = Vec::new();
for subspace in families.into_iter().flat_map(|f| f.subspaces()).copied() {
let name = format!("subspace_{}", store::subspace_name(subspace));
let (writer, upload) = match &sink {
Sink::Dir(dir) => (Output::File(dir.join(&name)), None),
Sink::S3(transfer) => {
println!("Exporting database to {}/{name}.", transfer.location());
let (writer, upload) = transfer.writer(&name);
(Output::S3(writer), Some(upload))
}
};
let (async_handle, sync_handle) = if subspace == SUBSPACE_BLOBS {
self.backup_blobs(&params.dest, subspace, schema_version)
self.backup_blobs(writer, subspace, schema_version)
} else {
self.backup_subspace(&params.dest, subspace, schema_version)
self.backup_subspace(writer, subspace, schema_version)
};
async_handle.await.failed("Task failed");
sync_handles.push(sync_handle);
if let Some(upload) = upload {
uploads.push((name, subspace, upload));
}
}
for handle in sync_handles {
handle.join().expect("Failed to join thread");
}
// inbuxa: K-C11: the uploads, then the manifest that says the
// export is whole
if let Sink::S3(transfer) = &sink {
let mut files = Vec::new();
let mut bytes = 0u64;
for (name, subspace, upload) in uploads {
let size = upload
.await
.failed("Upload task failed")
.unwrap_or_else(|err| failed(&format!("Export failed: {err}")));
bytes += size;
files.push(serde_json::json!({"name": name, "subspace": subspace, "bytes": size}));
}
let manifest = serde_json::json!({
"format": 1,
"version": types::brand_version!(),
"schemaVersion": schema_version,
"createdAt": store::write::now(),
"partial": families_given,
"files": files,
});
transfer
.put(
S3_EXPORT_MANIFEST,
&serde_json::to_vec_pretty(&manifest).unwrap_or_default(),
)
.await
.unwrap_or_else(|err| failed(&format!("Export failed: {err}")));
println!(
"Exported {} files, {bytes} bytes compressed, to {}.",
manifest["files"].as_array().map_or(0, Vec::len),
transfer.location()
);
}
}
fn backup_blobs(&self, dest: &Path, subspace: u8, schema_version: u32) -> TaskHandle {
fn backup_blobs(&self, output: Output, subspace: u8, schema_version: u32) -> TaskHandle {
let store = self.storage.data.clone();
let blob_store = self.storage.blob.clone();
let (handle, writer) = spawn_writer(
dest.join(format!("subspace_{}", store::subspace_name(subspace))),
subspace,
schema_version,
);
let (handle, writer) = spawn_writer(output, subspace, schema_version);
(
tokio::spawn(async move {
let mut blobs = Vec::new();
@@ -175,13 +246,9 @@ impl Core {
)
}
fn backup_subspace(&self, dest: &Path, subspace: u8, schema_version: u32) -> TaskHandle {
fn backup_subspace(&self, output: Output, subspace: u8, schema_version: u32) -> TaskHandle {
let store = self.storage.data.clone();
let (handle, writer) = spawn_writer(
dest.join(format!("subspace_{}", store::subspace_name(subspace))),
subspace,
schema_version,
);
let (handle, writer) = spawn_writer(output, subspace, schema_version);
(
tokio::spawn(async move {
if !store.is_sql() || (subspace != SUBSPACE_COUNTER && subspace != SUBSPACE_QUOTA) {
@@ -253,20 +320,60 @@ impl Core {
}
}
/// inbuxa: K-C11: one export file's destination
enum Output {
File(PathBuf),
S3(store::inbuxa_transfer::PartWriter),
}
/// inbuxa: K-C11: the sending end of an export file's writer. The export's
/// tasks send from the runtime's workers; when the writer is behind (an
/// upload to S3 waits on the runtime too), a plain blocking send would hold
/// the worker it needs, and with one worker (a pod limited to one CPU) the
/// export would stop there. A send that has to wait hands its worker over
/// first.
struct ExportSender(SyncSender<(Vec<u8>, Vec<u8>)>);
impl ExportSender {
fn send(&self, item: (Vec<u8>, Vec<u8>)) -> Result<(), mpsc::SendError<(Vec<u8>, Vec<u8>)>> {
match self.0.try_send(item) {
Ok(()) => Ok(()),
Err(mpsc::TrySendError::Full(item)) => {
if can_block_in_place() {
tokio::task::block_in_place(|| self.0.send(item))
} else {
self.0.send(item)
}
}
Err(mpsc::TrySendError::Disconnected(item)) => Err(mpsc::SendError(item)),
}
}
}
/// Whether this thread is a worker of a multi-threaded runtime, the only
/// kind `block_in_place` works on
fn can_block_in_place() -> bool {
tokio::runtime::Handle::try_current()
.is_ok_and(|h| h.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread)
}
#[allow(clippy::type_complexity)]
fn spawn_writer(
path: PathBuf,
output: Output,
subspace: u8,
version: u32,
) -> (std::thread::JoinHandle<()>, SyncSender<(Vec<u8>, Vec<u8>)>) {
) -> (std::thread::JoinHandle<()>, ExportSender) {
let (tx, rx) = mpsc::sync_channel::<(Vec<u8>, Vec<u8>)>(10);
let handle = std::thread::spawn(move || {
println!("Exporting database to {}.", path.to_str().unwrap());
let mut file = FrameEncoder::new(BufWriter::new(
std::fs::File::create(path).failed("Failed to create backup file"),
));
let out: Box<dyn Write + Send> = match output {
Output::File(path) => {
println!("Exporting database to {}.", path.to_str().unwrap());
Box::new(std::fs::File::create(path).failed("Failed to create backup file"))
}
Output::S3(writer) => Box::new(writer),
};
let mut file = FrameEncoder::new(BufWriter::new(out));
file.write_all(&[MAGIC_MARKER, subspace])
.failed("Failed to write version");
file.write_all(&version.to_le_bytes())
@@ -290,7 +397,7 @@ fn spawn_writer(
file.flush().failed("Failed to flush backup file");
});
(handle, tx)
(handle, ExportSender(tx))
}
impl BackupParams {
+43 -7
View File
@@ -42,6 +42,9 @@ pub struct BootManager {
/// inbuxa: K-S8: whether the store was new when this process started,
/// before the defaults went in
pub store_was_new: bool,
/// inbuxa: K-C11: started with `--store-check quick|full`: like
/// `--upgrade-plan`, then main runs a store check here and exits
pub store_check: Option<String>,
}
pub struct IpcReceivers {
@@ -61,13 +64,14 @@ Usage: inbuxa [OPTIONS]
Options:
-c, --config <PATH> Start server with the specified configuration file
-e, --export <PATH> Export all store data to a specific path
-i, --import <PATH> Import store data from a specific path
-e, --export <PATH> Export all store data to a directory or s3://bucket/prefix
-i, --import <PATH> Import store data from a directory or s3:// address
-o, --console Open the store console
-u, --upgrade-store Upgrade the store's layout, then exit
--seed <PATH> Set up a new store from a file, then exit
--upgrade-plan Print what --upgrade-store would do, then exit
--check Check the data, blob store and coordinator, then exit
--store-check <quick|full> Check every blob the store names, then exit
-h, --help Print help
-V, --version Print version
@@ -78,6 +82,20 @@ queues the index to be rebuilt when the server next starts. EXPORT_TYPES
limits an export to some of: data, registry, blob, changelog, queue, report,
telemetry, tasks.
An export to s3://bucket/prefix streams to a new folder there, named for the
time (20261119T204500Z), and writes export.json last: an export without it
didn't finish. An import from s3://bucket/prefix/<name> reads one back; from
s3://bucket/prefix, the newest finished one. INBUXA_EXPORT_S3_ENDPOINT names
an S3-compatible service (otherwise AWS), INBUXA_EXPORT_S3_REGION its region,
and INBUXA_EXPORT_S3_ACCESS_KEY_FILE and INBUXA_EXPORT_S3_SECRET_KEY_FILE
files holding its keys (otherwise the AWS_ variables or the instance's role).
--store-check runs a store check, as the console's Store checks page does:
quick finds blobs that are missing, full also reads each one and checks its
hash. It is saved with the others, and prints a summary. It exits 0 when it
found nothing wrong, 3 when it found problems (the console lists them),
2 when another check was running, 1 when it failed.
--upgrade-store does what a start does to the store before it serves, and
exits: 0 when the store is at this release's layout (whether it upgraded it
or found it there), 1 when the upgrade failed, 2 when it refused because
@@ -119,6 +137,8 @@ enum StoreOp {
UpgradePlan,
// inbuxa: K-C5
Check,
// inbuxa: K-C11
StoreCheck(String),
None,
}
@@ -150,7 +170,7 @@ impl BootManager {
};
let takes_value = matches!(
key.as_str(),
"config" | "c" | "export" | "e" | "import" | "i" | "seed"
"config" | "c" | "export" | "e" | "import" | "i" | "seed" | "store-check"
);
let value = if takes_value {
inline.or_else(|| args.next())
@@ -191,6 +211,10 @@ impl BootManager {
("seed", Some(value)) => {
import_export = StoreOp::Seed(value.into());
}
("store-check", Some(value)) => match value.as_str() {
"quick" | "full" => import_export = StoreOp::StoreCheck(value),
_ => failed("--store-check takes quick or full."),
},
(_, None) if takes_value => failed(&format!(
"Missing value for argument '{key}', try '--help'."
)),
@@ -223,7 +247,11 @@ impl BootManager {
};
// inbuxa: K-S8: a seed run boots as --upgrade-store does
let upgrade_plan = import_export == StoreOp::UpgradePlan;
let one_shot = upgrade_store || upgrade_plan || seed.is_some();
let store_check = match &import_export {
StoreOp::StoreCheck(mode) => Some(mode.clone()),
_ => None,
};
let one_shot = upgrade_store || upgrade_plan || seed.is_some() || store_check.is_some();
let registry = RegistryStore::init(
PathBuf::from(config_path.unwrap()),
import_export == StoreOp::None || one_shot,
@@ -232,8 +260,9 @@ impl BootManager {
.unwrap_or_else(|err| startup_failed(err));
if one_shot && (registry.is_bootstrap_mode() || registry.is_recovery_mode()) {
failed(
"--upgrade-store and --seed need a configured server: there is no data \
store in bootstrap mode, and recovery mode is for a person at the console.",
"--upgrade-store, --seed and --store-check need a configured server: there is \
no data store in bootstrap mode, and recovery mode is for a person at the \
console.",
);
}
let mut bootstrap = Bootstrap::new(registry).await;
@@ -267,7 +296,11 @@ impl BootManager {
let telemetry = Telemetry::parse(&mut bootstrap, &storage).await;
match import_export {
StoreOp::None | StoreOp::UpgradeStore | StoreOp::UpgradePlan | StoreOp::Seed(_) => {
StoreOp::None
| StoreOp::UpgradeStore
| StoreOp::UpgradePlan
| StoreOp::Seed(_)
| StoreOp::StoreCheck(_) => {
// Parse components
let core: Box<Core> =
Box::new(Box::pin(Core::parse(&mut bootstrap, storage)).await);
@@ -285,6 +318,8 @@ impl BootManager {
"Started with --upgrade-store: upgrading the store layout, then exiting"
} else if upgrade_plan {
"Started with --upgrade-plan: reporting what an upgrade would do, then exiting"
} else if store_check.is_some() {
"Started with --store-check: checking the store's blobs, then exiting"
} else {
"Started with --seed: setting up a new store from a file, then exiting"
},
@@ -381,6 +416,7 @@ impl BootManager {
upgrade_plan,
seed,
store_was_new,
store_check,
}
}
StoreOp::Export(path) => {
+210 -18
View File
@@ -6,7 +6,7 @@
* Modified by Coffey Labs in 2026 for INBUXA.
*/
use super::backup::MAGIC_MARKER;
use super::backup::{MAGIC_MARKER, S3_EXPORT_MANIFEST};
use crate::{Core, DATABASE_SCHEMA_VERSION};
use lz4_flex::frame::FrameDecoder;
use registry::{
@@ -22,7 +22,7 @@ use std::{
path::{Path, PathBuf},
};
use store::{
BlobStore, IterateParams, SUBSPACE_BLOBS, SUBSPACE_COUNTER, SUBSPACE_INDEXES,
BlobStore, IterateParams, SUBSPACE_BLOBS, SUBSPACE_COUNTER, SUBSPACE_INBUXA, SUBSPACE_INDEXES,
SUBSPACE_PROPERTY, SUBSPACE_QUOTA, SUBSPACE_REGISTRY_PK, SUBSPACE_TELEMETRY_SPAN, Store,
U32_LEN,
write::{
@@ -37,6 +37,12 @@ impl Core {
/// Imports an export into an empty store and returns the subspaces it
/// wrote. inbuxa: the caller hands them to [`Core::queue_reindex`].
pub async fn restore(&self, src: PathBuf) -> Vec<u8> {
// inbuxa: K-C11: an export in S3
let src_text = src.to_string_lossy().to_string();
if store::inbuxa_transfer::is_s3_url(&src_text) {
return self.restore_s3(&src_text).await;
}
// Backup the core
let paths = if src.is_dir() {
let mut paths = Vec::new();
@@ -52,11 +58,91 @@ impl Core {
vec![src]
};
let sources = paths.into_iter().map(Source::File).collect::<Vec<_>>();
self.restore_sources(sources).await
}
/// inbuxa: K-C11: `s3://bucket/prefix/<name>`, or the newest finished
/// export under `s3://bucket/prefix`. An export without its manifest
/// didn't finish, and is refused.
async fn restore_s3(&self, url: &str) -> Vec<u8> {
use store::inbuxa_transfer::S3Transfer;
let base =
S3Transfer::open(url).unwrap_or_else(|err| failed(&format!("Import failed: {err}")));
let (transfer, manifest) = match base.get(S3_EXPORT_MANIFEST).await {
Ok(Some(manifest)) => (base, manifest),
Ok(None) => {
let names = base
.children_with(S3_EXPORT_MANIFEST)
.await
.unwrap_or_else(|err| failed(&format!("Import failed: {err}")));
let Some(newest) = names.last() else {
failed(&format!(
"Import failed: no finished export at {url} or under it (an export is \
finished once its {S3_EXPORT_MANIFEST} is written)."
));
};
let transfer = base.join(newest);
println!(
"Importing the newest finished export, {}.",
transfer.location()
);
let manifest = transfer
.get(S3_EXPORT_MANIFEST)
.await
.ok()
.flatten()
.unwrap_or_else(|| failed("Import failed: the export's manifest went away"));
(transfer, manifest)
}
Err(err) => failed(&format!("Import failed: {err}")),
};
let manifest: serde_json::Value = serde_json::from_slice(&manifest)
.unwrap_or_else(|err| failed(&format!("Import failed: {S3_EXPORT_MANIFEST}: {err}")));
let sources = manifest["files"]
.as_array()
.unwrap_or_else(|| {
failed(&format!(
"Import failed: {S3_EXPORT_MANIFEST} lists no files"
))
})
.iter()
.map(|file| {
let name = file["name"].as_str().unwrap_or_default();
let subspace = file["subspace"].as_u64().and_then(|s| u8::try_from(s).ok());
match (name, subspace) {
(name, Some(subspace))
if name.starts_with("subspace_") && !name.contains('/') =>
{
Source::S3 {
transfer: transfer.clone(),
name: name.to_string(),
subspace,
}
}
_ => failed(&format!(
"Import failed: {S3_EXPORT_MANIFEST}: a bad file entry: {file}"
)),
}
})
.collect::<Vec<_>>();
self.restore_sources(sources).await
}
async fn restore_sources(&self, paths: Vec<Source>) -> Vec<u8> {
let mut conflicts = Vec::new();
for path in &paths {
let subspace = KeyValueReader::new(path).subspace;
let subspace = path.subspace();
// inbuxa: K-C11: every start writes the fork's own subspace, this
// import's too: where the blob store is (W), the permission
// grants (P), the log files' setting (T). Its records are keyed
// values, which an import sets over, never duplicates; a store in
// use still has data in the other subspaces, which are checked.
if subspace == SUBSPACE_INBUXA {
continue;
}
if subspace_has_data(&self.storage.data, subspace).await {
conflicts.push(path.clone());
conflicts.push(path.describe());
}
}
@@ -68,34 +154,85 @@ impl Core {
start the server before importing. Conflicting dumps:"
);
for path in conflicts {
eprintln!(" {}", path.display());
eprintln!(" {path}");
}
std::process::exit(1);
}
let mut imported = paths
.iter()
.map(|path| KeyValueReader::new(path).subspace)
.collect::<Vec<_>>();
let mut imported = paths.iter().map(Source::subspace).collect::<Vec<_>>();
imported.sort_unstable();
imported.dedup();
// inbuxa: K-C11: the blobs last, into the blob store the imported
// settings name. This process started on an empty store, whose
// blob store is the data store; a server started on the imported
// one reads blobs from wherever its x:BlobStore says (S3, say), so
// blobs written to the data store would be missing there.
let (blobs, rest): (Vec<_>, Vec<_>) = paths
.into_iter()
.partition(|path| path.subspace() == SUBSPACE_BLOBS);
let mut tasks = Vec::new();
for path in paths {
for path in rest {
let storage = self.storage.clone();
let blob_store = self.storage.blob.clone();
tasks.push(tokio::spawn(async move {
restore_file(storage.data, blob_store, &path).await;
}));
}
for task in tasks {
task.await.failed("Failed to wait for task");
}
if !blobs.is_empty() {
let blob_store = self.imported_blob_store().await;
let mut tasks = Vec::new();
for path in blobs {
let storage = self.storage.clone();
let blob_store = blob_store.clone();
tasks.push(tokio::spawn(async move {
restore_file(storage.data, blob_store, &path).await;
}));
}
for task in tasks {
task.await.failed("Failed to wait for task");
}
}
imported
}
/// inbuxa: K-C11: the blob store the store's x:BlobStore names now,
/// after the import wrote it
async fn imported_blob_store(&self) -> BlobStore {
use registry::{schema::structs::BlobStore as BlobStoreSetting, types::ObjectImpl};
let setting = self
.storage
.registry
.object::<BlobStoreSetting>(BlobStoreSetting::OBJECT.singleton().id())
.await
.failed("Failed to read the imported blob store setting")
.unwrap_or_default();
if matches!(setting, BlobStoreSetting::Default) {
println!("Importing blobs into the data store, as the export's settings say.");
return self.storage.data.clone().into();
}
let mut warnings = Vec::new();
let store = BlobStore::open(setting, &self.storage.data, &mut warnings)
.await
.unwrap_or_else(|err| {
failed(&format!(
"Can't open the blob store the export's settings name, to import the \
blobs into: {err}"
))
});
for warning in warnings {
eprintln!("Blob store: {warning}");
}
println!("Importing blobs into the blob store the export's settings name.");
store
}
/// inbuxa: an export never carries the full-text index. It is built by
/// and for one search backend (the SQL stores index into their own
/// tables, the key-value stores into a subspace, external engines keep it
@@ -172,10 +309,59 @@ async fn subspace_has_data(store: &Store, subspace: u8) -> bool {
has_data
}
async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) {
println!("Importing database dump from {}.", path.to_str().unwrap());
/// inbuxa: K-C11: one export file: on disk, or in S3
enum Source {
File(PathBuf),
S3 {
transfer: store::inbuxa_transfer::S3Transfer,
name: String,
/// From the manifest, so the file needn't be opened to know it
subspace: u8,
},
}
let mut reader = KeyValueReader::new(path);
impl Source {
fn subspace(&self) -> u8 {
match self {
Source::File(path) => KeyValueReader::new(path).subspace,
Source::S3 { subspace, .. } => *subspace,
}
}
fn describe(&self) -> String {
match self {
Source::File(path) => path.display().to_string(),
Source::S3 { transfer, name, .. } => format!("{}/{name}", transfer.location()),
}
}
fn open(&self) -> KeyValueReader {
match self {
Source::File(path) => KeyValueReader::new(path),
Source::S3 {
transfer,
name,
subspace,
} => {
let reader =
KeyValueReader::from_reader(Box::new(transfer.reader(name)), &self.describe());
if reader.subspace != *subspace {
failed(&format!(
"{} holds subspace {}, but the manifest says {subspace}",
self.describe(),
reader.subspace
));
}
reader
}
}
}
}
async fn restore_file(store: Store, blob_store: BlobStore, path: &Source) {
println!("Importing database dump from {}.", path.describe());
let mut reader = path.open();
let mut batch = BatchBuilder::new();
match reader.subspace {
@@ -276,14 +462,20 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) {
struct KeyValueReader {
subspace: u8,
file: FrameDecoder<BufReader<File>>,
file: FrameDecoder<BufReader<Box<dyn Read + Send>>>,
}
impl KeyValueReader {
fn new(path: &Path) -> Self {
let mut file = FrameDecoder::new(BufReader::new(
File::open(path).failed("Failed to open file"),
));
Self::from_reader(
Box::new(File::open(path).failed("Failed to open file")),
&path.display().to_string(),
)
}
// inbuxa: K-C11: any reader: a file, or a download from S3
fn from_reader(reader: Box<dyn Read + Send>, path: &str) -> Self {
let mut file = FrameDecoder::new(BufReader::new(reader));
let mut buf = [0u8; 1];
file.read_exact(&mut buf)
.failed(&format!("Failed to read magic marker from {path:?}"));
+60
View File
@@ -80,6 +80,12 @@ async fn main() -> std::io::Result<()> {
std::process::exit(Box::pin(seed_and_exit(&server, &path, init.store_was_new)).await);
}
// inbuxa: K-C11: --store-check runs a check and exits
if let Some(mode) = init.store_check.clone() {
let server = init.inner.build_server();
std::process::exit(Box::pin(store_check_and_exit(&server, &mode)).await);
}
// Migrate database
if let Err(err) = migration::try_migrate(&init.inner.build_server()).await {
trc::event!(
@@ -274,6 +280,60 @@ async fn seed_and_exit(
flush_and_exit_code(code).await
}
/// inbuxa: K-C11: `--store-check`: 0 found nothing wrong, 3 found problems,
/// 2 another check was running, 1 failed
async fn store_check_and_exit(server: &common::Server, mode: &str) -> i32 {
use services::inbuxa_store_check::{
CheckMode as Mode, CheckRefusal as Refusal, CheckStatus as Status,
};
let mode = Mode::parse(mode).unwrap_or(Mode::Quick);
let (code, message) = match services::inbuxa_store_check::run_now(server, mode).await {
Ok(Ok(check)) => {
let p = &check.progress;
let summary = format!(
"Store check {} ({}): {} blobs, {} links, {} messages checked, {} bytes read. \
Missing {}, damaged {}, unlinked {}, unreadable {}.",
check.id,
mode.as_str(),
p.blobs_checked,
p.links_checked,
p.messages_checked,
p.bytes_read,
p.missing,
p.damaged,
p.unlinked,
p.unreadable,
);
match check.status {
Status::Done if p.problems() == 0 => (0, format!("{summary} Nothing wrong.")),
Status::Done => (
3,
format!(
"{summary} {} problems: the console's Store checks page lists them.",
p.problems()
),
),
status => (
1,
format!(
"{summary} It ended {}: {}",
status.as_str(),
check.error.as_deref().unwrap_or("no reason given")
),
),
}
}
Ok(Err(Refusal::Running(id))) => (
2,
format!("Store check {id} is running: one check runs at a time."),
),
Err(err) => (1, format!("The store check failed: {err}")),
};
println!("{message}");
flush_and_exit_code(code).await
}
/// inbuxa: K-C9: `--upgrade-plan`: one line of JSON on stdout, for a
/// script to read, and the exit code a shell can branch on
async fn upgrade_plan_and_exit(server: &common::Server) -> i32 {
+29
View File
@@ -74,6 +74,35 @@ pub async fn start(
Ok(Ok(id))
}
// inbuxa: K-C11: for main's --store-check
pub use inbuxa_features::store_check::{
Mode as CheckMode, Refusal as CheckRefusal, Status as CheckStatus,
};
/// inbuxa: K-C11: the node id a `--store-check` run records. Leased node
/// ids are 16-bit, so no server's start-up (SV-8) takes it for its own.
pub const ONE_SHOT_NODE_ID: u64 = 1 << 32;
/// inbuxa: K-C11: `--store-check`: a check run here, to its end, for a
/// scheduled job. Saved like any other, so the console shows it.
pub async fn run_now(server: &Server, mode: Mode) -> trc::Result<Result<StoreCheck, Refusal>> {
let now = now();
let check = StoreCheck::new(mode, "store-check job".to_string(), ONE_SHOT_NODE_ID, now);
let id = match model::create(server.store(), &check, now).await? {
Ok(id) => id,
Err(refusal) => return Ok(Err(refusal)),
};
run(server, StoreCheck { id, ..check }).await;
Ok(model::get(server.store(), id)
.await?
.ok_or_else(|| {
trc::StoreEvent::NotFound
.into_err()
.details("The check went away")
})
.map(Ok)?)
}
/// SV-8: asks a running check to stop. Any node can: the one running it
/// sees the request at its next save. None if there's no such check.
pub async fn stop(server: &Server, id: u32) -> trc::Result<Option<StoreCheck>> {
+1 -1
View File
@@ -65,7 +65,7 @@ foundation = ["foundationdb", "futures"]
fdb-chunked-bm = []
# Blob stores
s3 = ["rust-s3", "dep:reqwest_s3"]
s3 = ["rust-s3", "dep:reqwest_s3", "futures"]
azure = ["azure_core", "azure_storage", "azure_storage_blobs", "futures"]
# In-memory stores
+592
View File
@@ -0,0 +1,592 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
//! inbuxa: K-C11: an export written to, and read back from, an S3 bucket.
//!
//! `--export s3://bucket/prefix` streams each export file into a multipart
//! upload, so nothing the size of the store lands on local disk, and
//! `--import s3://bucket/prefix/<name>` streams it back. The bucket's
//! address and keys come from the environment:
//!
//! - `INBUXA_EXPORT_S3_ENDPOINT`: an S3-compatible service (MinIO, Garage,
//! Ceph...), addressed path-style. Without it, AWS.
//! - `INBUXA_EXPORT_S3_REGION`: `us-east-1` unless given.
//! - `INBUXA_EXPORT_S3_ACCESS_KEY_FILE`, `INBUXA_EXPORT_S3_SECRET_KEY_FILE`:
//! files holding the keys. Without them, the usual AWS sources (the
//! `AWS_` variables, a profile, the instance's role).
//!
//! The export and import code is sync (a writer thread per file, readers in
//! the import tasks); these bridge it to the async S3 client.
/// Whether a `--export`/`--import` value names a bucket.
pub fn is_s3_url(value: &str) -> bool {
value.starts_with("s3://")
}
/// The name an export made now goes under: `20261119T204500Z`.
pub fn export_name(now: u64) -> String {
let days = (now / 86_400) as i64;
let secs = now % 86_400;
// Howard Hinnant's civil_from_days
let z = days + 719_468;
let era = z.div_euclid(146_097);
let doe = z.rem_euclid(146_097);
let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
let mp = (5 * doy + 2) / 153;
let day = doy - (153 * mp + 2) / 5 + 1;
let month = if mp < 10 { mp + 3 } else { mp - 9 };
let year = yoe + era * 400 + i64::from(month <= 2);
format!(
"{year:04}{month:02}{day:02}T{:02}{:02}{:02}Z",
secs / 3600,
(secs / 60) % 60,
secs % 60
)
}
/// The size of upload part `n` (from 1): 16 MiB for the first thousand,
/// doubling every thousand after, up to 512 MiB. S3 takes at most 10,000
/// parts, so one file can be nearly 3 TiB, and only one part is held at a
/// time.
pub fn part_size(part: u32) -> usize {
const MIB: usize = 1024 * 1024;
let tier = (part.saturating_sub(1) / 1000).min(5);
(16 * MIB) << tier
}
/// The most parts S3 takes in one upload.
pub const MAX_PARTS: u32 = 10_000;
#[cfg(feature = "s3")]
pub use imp::{PartReader, PartWriter, S3Transfer};
#[cfg(not(feature = "s3"))]
pub use none::{PartReader, PartWriter, S3Transfer};
#[cfg(feature = "s3")]
mod imp {
use super::{MAX_PARTS, part_size};
use futures::StreamExt;
use s3::{Bucket, Region, creds::Credentials, serde_types::Part};
use std::io::{Read, Write};
use std::sync::mpsc as std_mpsc;
use tokio::sync::mpsc;
const CONTENT_TYPE: &str = "application/octet-stream";
const RETRIES: u32 = 5;
#[derive(Clone)]
pub struct S3Transfer {
bucket: Box<Bucket>,
bucket_name: String,
/// No leading or trailing slash; may be empty.
prefix: String,
}
fn env(name: &str) -> Option<String> {
std::env::var(name)
.ok()
.map(|v| v.trim().to_string())
.filter(|v| !v.is_empty())
}
fn key_file(name: &str) -> Result<Option<String>, String> {
match env(name) {
Some(path) => std::fs::read_to_string(&path)
.map(|v| Some(v.trim_end_matches(['\r', '\n']).to_string()))
.map_err(|err| format!("{name}: can't read {path}: {err}")),
None => Ok(None),
}
}
impl S3Transfer {
/// `s3://bucket/prefix`, with the bucket's address and keys from the
/// environment.
pub fn open(url: &str) -> Result<Self, String> {
let rest = url
.strip_prefix("s3://")
.ok_or_else(|| format!("{url}: not an s3:// address"))?;
let (bucket_name, prefix) = rest.split_once('/').unwrap_or((rest, ""));
if bucket_name.is_empty() {
return Err(format!("{url}: no bucket named"));
}
let region_name = env("INBUXA_EXPORT_S3_REGION").unwrap_or_else(|| "us-east-1".into());
let endpoint = env("INBUXA_EXPORT_S3_ENDPOINT");
let region = match &endpoint {
Some(endpoint) => Region::Custom {
region: region_name,
endpoint: endpoint.trim_end_matches('/').to_string(),
},
None => region_name
.parse()
.map_err(|err| format!("INBUXA_EXPORT_S3_REGION: {err}"))?,
};
let access = key_file("INBUXA_EXPORT_S3_ACCESS_KEY_FILE")?;
let secret = key_file("INBUXA_EXPORT_S3_SECRET_KEY_FILE")?;
let credentials = match (&access, &secret) {
(Some(access), Some(secret)) => {
Credentials::new(Some(access), Some(secret), None, None, None)
}
(None, None) => Credentials::default(),
_ => {
return Err("INBUXA_EXPORT_S3_ACCESS_KEY_FILE and \
INBUXA_EXPORT_S3_SECRET_KEY_FILE go together"
.into());
}
}
.map_err(|err| format!("S3 credentials: {err}"))?;
let mut bucket = Bucket::new(bucket_name, region, credentials)
.map_err(|err| format!("S3 bucket {bucket_name}: {err}"))?;
if endpoint.is_some() {
bucket = bucket.with_path_style();
}
Ok(S3Transfer {
bucket,
bucket_name: bucket_name.to_string(),
prefix: prefix.trim_matches('/').to_string(),
})
}
/// The same bucket, one level down.
pub fn join(&self, name: &str) -> Self {
S3Transfer {
prefix: self.key(name),
..self.clone()
}
}
pub fn location(&self) -> String {
format!("s3://{}/{}", self.bucket_name, self.prefix)
}
fn key(&self, name: &str) -> String {
if self.prefix.is_empty() {
name.to_string()
} else {
format!("{}/{name}", self.prefix)
}
}
/// A small object, whole.
pub async fn put(&self, name: &str, data: &[u8]) -> Result<(), String> {
let key = self.key(name);
let mut last = String::new();
for attempt in 0..RETRIES {
match self.bucket.put_object(&key, data).await {
Ok(r) if (200..300).contains(&r.status_code()) => return Ok(()),
Ok(r) => {
last = format!(
"{}: {}",
r.status_code(),
String::from_utf8_lossy(r.as_slice())
)
}
Err(err) => last = err.to_string(),
}
wait(attempt).await;
}
Err(format!(
"Failed to write {}: {last}",
self.location_of(name)
))
}
/// A small object, whole; None when it isn't there.
pub async fn get(&self, name: &str) -> Result<Option<Vec<u8>>, String> {
let key = self.key(name);
let mut last = String::new();
for attempt in 0..RETRIES {
match self.bucket.get_object(&key).await {
Ok(r) if (200..300).contains(&r.status_code()) => {
return Ok(Some(r.to_vec()));
}
Ok(r) if r.status_code() == 404 => return Ok(None),
Ok(r) => last = format!("{}", r.status_code()),
Err(s3::error::S3Error::HttpFailWithBody(404, _)) => return Ok(None),
Err(err) => last = err.to_string(),
}
wait(attempt).await;
}
Err(format!("Failed to read {}: {last}", self.location_of(name)))
}
/// The names one level down that hold `marker`: the exports under a
/// prefix.
pub async fn children_with(&self, marker: &str) -> Result<Vec<String>, String> {
let prefix = if self.prefix.is_empty() {
String::new()
} else {
format!("{}/", self.prefix)
};
let results = self
.bucket
.list(prefix.clone(), None)
.await
.map_err(|err| format!("Failed to list {}: {err}", self.location()))?;
let mut names = results
.iter()
.flat_map(|r| r.contents.iter())
.filter_map(|o| o.key.strip_prefix(&prefix))
.filter_map(|rest| rest.strip_suffix(marker)?.strip_suffix('/'))
.filter(|name| !name.is_empty() && !name.contains('/'))
.map(str::to_string)
.collect::<Vec<_>>();
names.sort();
Ok(names)
}
fn location_of(&self, name: &str) -> String {
format!("s3://{}/{}", self.bucket_name, self.key(name))
}
/// A writer whose bytes go up as a multipart upload under `name`,
/// for a thread outside the runtime; the upload runs on the
/// runtime. Dropping the writer sends the last part; the handle
/// ends when the upload is complete.
pub fn writer(
&self,
name: &str,
) -> (PartWriter, tokio::task::JoinHandle<Result<u64, String>>) {
let (tx, rx) = mpsc::channel::<Vec<u8>>(1);
let this = self.clone();
let name = name.to_string();
let handle = tokio::spawn(async move { this.upload(&name, rx).await });
(
PartWriter {
tx: Some(tx),
buf: Vec::with_capacity(part_size(1)),
part: 1,
},
handle,
)
}
async fn upload(&self, name: &str, mut rx: mpsc::Receiver<Vec<u8>>) -> Result<u64, String> {
let key = self.key(name);
let location = self.location_of(name);
let upload_id = self
.bucket
.initiate_multipart_upload(&key, CONTENT_TYPE)
.await
.map_err(|err| format!("Failed to start uploading {location}: {err}"))?
.upload_id;
let mut parts: Vec<Part> = Vec::new();
let mut total = 0u64;
let result: Result<(), String> = async {
while let Some(chunk) = rx.recv().await {
let number = parts.len() as u32 + 1;
if number > MAX_PARTS {
return Err(format!("{location}: too large for one upload"));
}
total += chunk.len() as u64;
parts.push(
self.put_part(chunk, &key, number, &upload_id, &location)
.await?,
);
}
Ok(())
}
.await;
if let Err(err) = result {
// Dropping rx fails the writer's next send, which stops it
drop(rx);
let _ = self.bucket.abort_upload(&key, &upload_id).await;
return Err(err);
}
let mut last = String::new();
for attempt in 0..RETRIES {
match self
.bucket
.complete_multipart_upload(&key, &upload_id, parts.clone())
.await
{
Ok(r) if (200..300).contains(&r.status_code()) => return Ok(total),
Ok(r) => last = format!("{}", r.status_code()),
Err(err) => last = err.to_string(),
}
wait(attempt).await;
}
let _ = self.bucket.abort_upload(&key, &upload_id).await;
Err(format!("Failed to finish uploading {location}: {last}"))
}
async fn put_part(
&self,
chunk: Vec<u8>,
key: &str,
number: u32,
upload_id: &str,
location: &str,
) -> Result<Part, String> {
let mut last = String::new();
for attempt in 0..RETRIES {
match self
.bucket
.put_multipart_chunk(chunk.clone(), key, number, upload_id, CONTENT_TYPE)
.await
{
Ok(part) => return Ok(part),
Err(err) => last = err.to_string(),
}
wait(attempt).await;
}
Err(format!(
"Failed to upload part {number} of {location}: {last}"
))
}
/// A reader of the object `name`, for code that reads synchronously
/// on the runtime's workers: the download is a task on the runtime,
/// and a reader that has to wait for it hands its worker over.
pub fn reader(&self, name: &str) -> PartReader {
let (tx, rx) = std_mpsc::sync_channel::<Result<Vec<u8>, String>>(4);
let this = self.clone();
let key = self.key(name);
let location = self.location_of(name);
tokio::spawn(async move {
{
let mut stream = match this.bucket.get_object_stream(&key).await {
Ok(stream) if (200..300).contains(&stream.status_code) => stream,
Ok(stream) => {
let _ = send_chunk(
&tx,
Err(format!("Failed to read {location}: {}", stream.status_code)),
);
return;
}
Err(err) => {
let _ =
send_chunk(&tx, Err(format!("Failed to read {location}: {err}")));
return;
}
};
while let Some(chunk) = stream.bytes().next().await {
let sent = match chunk {
Ok(bytes) => send_chunk(&tx, Ok(bytes.to_vec())),
Err(err) => {
let _ = send_chunk(
&tx,
Err(format!("Failed to read {location}: {err}")),
);
return;
}
};
if sent.is_err() {
return;
}
}
}
});
PartReader {
rx,
buf: Vec::new(),
pos: 0,
}
}
}
/// Whether this thread is a worker of a multi-threaded runtime, where a
/// wait can hand the worker over (`block_in_place`)
fn on_worker() -> bool {
tokio::runtime::Handle::try_current()
.is_ok_and(|h| h.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread)
}
/// A send that, when it has to wait for the reader, hands the worker over
fn send_chunk(
tx: &std_mpsc::SyncSender<Result<Vec<u8>, String>>,
item: Result<Vec<u8>, String>,
) -> Result<(), ()> {
match tx.try_send(item) {
Ok(()) => Ok(()),
Err(std_mpsc::TrySendError::Disconnected(_)) => Err(()),
Err(std_mpsc::TrySendError::Full(item)) if on_worker() => {
tokio::task::block_in_place(|| tx.send(item)).map_err(|_| ())
}
Err(std_mpsc::TrySendError::Full(item)) => tx.send(item).map_err(|_| ()),
}
}
async fn wait(attempt: u32) {
tokio::time::sleep(std::time::Duration::from_millis(250 << attempt.min(5))).await;
}
/// See [`S3Transfer::writer`].
pub struct PartWriter {
tx: Option<mpsc::Sender<Vec<u8>>>,
buf: Vec<u8>,
part: u32,
}
impl PartWriter {
fn send(&mut self) -> std::io::Result<()> {
let next = part_size(self.part + 1);
let chunk = std::mem::replace(&mut self.buf, Vec::with_capacity(next));
self.part += 1;
match &self.tx {
Some(tx) => tx.blocking_send(chunk).map_err(|_| {
std::io::Error::other("the upload stopped: its error is on the next line")
}),
None => Err(std::io::Error::other("the upload is finished")),
}
}
}
impl Write for PartWriter {
fn write(&mut self, data: &[u8]) -> std::io::Result<usize> {
let room = part_size(self.part) - self.buf.len();
let take = room.min(data.len());
self.buf.extend_from_slice(&data[..take]);
if self.buf.len() == part_size(self.part) {
self.send()?;
}
Ok(take)
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
impl Drop for PartWriter {
fn drop(&mut self) {
// The last part, which may be short; even an empty file is one
// part
if !self.buf.is_empty() || self.part == 1 {
let _ = self.send();
}
self.tx = None;
}
}
/// See [`S3Transfer::reader`].
pub struct PartReader {
rx: std_mpsc::Receiver<Result<Vec<u8>, String>>,
buf: Vec<u8>,
pos: usize,
}
impl Read for PartReader {
fn read(&mut self, out: &mut [u8]) -> std::io::Result<usize> {
while self.pos == self.buf.len() {
// Readers run on the runtime's workers (the import's tasks),
// and the download needs one to make progress: a wait hands
// this worker over first, or a pod with one CPU's worth of
// workers would stop here
let next = match self.rx.try_recv() {
Ok(next) => Ok(next),
Err(std_mpsc::TryRecvError::Disconnected) => Err(std_mpsc::RecvError),
Err(std_mpsc::TryRecvError::Empty) => {
if on_worker() {
tokio::task::block_in_place(|| self.rx.recv())
} else {
self.rx.recv()
}
}
};
match next {
Ok(Ok(chunk)) => {
self.buf = chunk;
self.pos = 0;
}
Ok(Err(err)) => return Err(std::io::Error::other(err)),
// The download ended
Err(_) => return Ok(0),
}
}
let n = out.len().min(self.buf.len() - self.pos);
out[..n].copy_from_slice(&self.buf[self.pos..self.pos + n]);
self.pos += n;
Ok(n)
}
}
}
#[cfg(not(feature = "s3"))]
mod none {
//! This build has no S3 client: [`S3Transfer::open`] always refuses, so
//! nothing else here is ever reached.
#[derive(Clone)]
pub struct S3Transfer;
pub struct PartWriter;
pub struct PartReader;
impl S3Transfer {
pub fn open(_url: &str) -> Result<Self, String> {
Err("this build has no S3 support (the s3 feature)".into())
}
pub fn join(&self, _name: &str) -> Self {
unreachable!()
}
pub fn location(&self) -> String {
unreachable!()
}
pub async fn put(&self, _name: &str, _data: &[u8]) -> Result<(), String> {
unreachable!()
}
pub async fn get(&self, _name: &str) -> Result<Option<Vec<u8>>, String> {
unreachable!()
}
pub async fn children_with(&self, _marker: &str) -> Result<Vec<String>, String> {
unreachable!()
}
pub fn writer(
&self,
_name: &str,
) -> (PartWriter, tokio::task::JoinHandle<Result<u64, String>>) {
unreachable!()
}
pub fn reader(&self, _name: &str) -> PartReader {
unreachable!()
}
}
impl std::io::Write for PartWriter {
fn write(&mut self, _data: &[u8]) -> std::io::Result<usize> {
unreachable!()
}
fn flush(&mut self) -> std::io::Result<()> {
unreachable!()
}
}
impl std::io::Read for PartReader {
fn read(&mut self, _out: &mut [u8]) -> std::io::Result<usize> {
unreachable!()
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn export_names_sort_by_time() {
assert_eq!(export_name(0), "19700101T000000Z");
assert_eq!(export_name(1_795_552_496), "20261124T203456Z");
assert_eq!(export_name(951_782_400), "20000229T000000Z");
assert!(export_name(1_795_552_496) < export_name(1_795_552_497));
}
#[test]
fn parts_grow_to_nearly_three_tebibytes() {
assert_eq!(part_size(1), 16 << 20);
assert_eq!(part_size(1000), 16 << 20);
assert_eq!(part_size(1001), 32 << 20);
assert_eq!(part_size(MAX_PARTS), 512 << 20);
let total: u64 = (1..=MAX_PARTS).map(|n| part_size(n) as u64).sum();
assert!(total > (29u64 << 40) / 10, "{total}");
}
#[test]
fn s3_urls() {
assert!(is_s3_url("s3://backups/inbuxa"));
assert!(!is_s3_url("/var/backups"));
}
}
+2
View File
@@ -11,6 +11,8 @@
pub mod backend;
pub mod build;
pub mod dispatch;
// inbuxa: K-C11: exports to and from S3
pub mod inbuxa_transfer;
pub mod query;
pub mod registry;
pub mod search;
+35
View File
@@ -278,6 +278,31 @@ pub async fn test(test: &TestServer) {
.backup(BackupParams::new(temp_dir.path.clone()))
.await;
// inbuxa: K-C11: a record in the fork's subspace, which every start
// writes (here whichever one the test server wrote first)
let mut startup_key = None;
db.iterate(
IterateParams::new(
AnyKey {
subspace: SUBSPACE_INBUXA,
key: vec![0u8],
},
AnyKey {
subspace: SUBSPACE_INBUXA,
key: vec![u8::MAX; 32],
},
)
.no_values(),
|key, _| {
startup_key = Some(key.to_vec());
Ok(false)
},
)
.await
.unwrap();
let startup_key =
startup_key.expect("the test server's start wrote nothing to the fork's subspace");
// Destroy store
println!("Destroying store...");
store_destroy(&db).await;
@@ -293,6 +318,16 @@ pub async fn test(test: &TestServer) {
.write("localhost")
.finalize(),
);
// inbuxa: K-C11: and over what any start writes first, the import's own
// too: a record in the fork's subspace, with a value the import has to
// set over. Refused as data in the store until rc.34.
batch.set(
ValueClass::Any(AnyClass {
subspace: SUBSPACE_INBUXA,
key: startup_key,
}),
b"written at start-up".to_vec(),
);
db.write(batch.build_all()).await.unwrap();
for (key, _) in &named_blobs {
test.server.blob_store().delete_blob(key).await.unwrap();