Backups for Kubernetes: export to and import from S3, store checks as a command (K-C11) #491
No files matched your search
@@ -11,7 +11,7 @@ jobs:
|
||||
announce:
|
||||
runs-on: light
|
||||
steps:
|
||||
- uses: coffey-labs/actions/discourse-release@ba4cde7c3b47cc9f93c0080da8ba838a1ef1845b
|
||||
- uses: coffey-labs/actions/discourse-release@958b3a35a345744abd236ca5abf679cbedaca7a0
|
||||
with:
|
||||
api-key: ${{ secrets.DISCOURSE_RELEASE_KEY }}
|
||||
discord-webhook: ${{ secrets.DISCORD_RELEASE_WEBHOOK }}
|
||||
@@ -511,7 +511,7 @@ jobs:
|
||||
if: ${{ always() && ((needs.release.result == 'success' && needs.binaries.result == 'success') || needs.github.result == 'success') }}
|
||||
runs-on: light
|
||||
steps:
|
||||
- uses: coffey-labs/actions/discourse-release@ba4cde7c3b47cc9f93c0080da8ba838a1ef1845b
|
||||
- uses: coffey-labs/actions/discourse-release@958b3a35a345744abd236ca5abf679cbedaca7a0
|
||||
with:
|
||||
api-key: ${{ secrets.DISCOURSE_RELEASE_KEY }}
|
||||
discord-webhook: ${{ secrets.DISCORD_RELEASE_WEBHOOK }}
|
||||
|
||||
@@ -11,7 +11,7 @@ use ahash::AHashSet;
|
||||
use lz4_flex::frame::FrameEncoder;
|
||||
use std::{
|
||||
io::{BufWriter, Write},
|
||||
path::{Path, PathBuf},
|
||||
path::PathBuf,
|
||||
sync::mpsc::{self, SyncSender},
|
||||
};
|
||||
use store::{
|
||||
@@ -19,7 +19,7 @@ use store::{
|
||||
*,
|
||||
};
|
||||
use types::blob_hash::{BLOB_HASH_LEN, BlobHash};
|
||||
use utils::{UnwrapFailure, codec::leb128::Leb128_};
|
||||
use utils::{UnwrapFailure, codec::leb128::Leb128_, failed};
|
||||
|
||||
pub(super) const MAGIC_MARKER: u8 = 123;
|
||||
|
||||
@@ -52,9 +52,31 @@ pub struct BackupParams {
|
||||
families: AHashSet<Family>,
|
||||
}
|
||||
|
||||
/// inbuxa: K-C11: where the export files go
|
||||
enum Sink {
|
||||
Dir(PathBuf),
|
||||
S3(store::inbuxa_transfer::S3Transfer),
|
||||
}
|
||||
|
||||
type Upload = tokio::task::JoinHandle<Result<u64, String>>;
|
||||
|
||||
/// inbuxa: K-C11: an export in S3 ends with this, written last, naming its
|
||||
/// files: an export without one didn't finish.
|
||||
pub const S3_EXPORT_MANIFEST: &str = "export.json";
|
||||
|
||||
impl Core {
|
||||
pub async fn backup(&self, mut params: BackupParams) {
|
||||
if !params.dest.exists() {
|
||||
// inbuxa: K-C11: s3://bucket/prefix exports to a new folder there
|
||||
let dest_text = params.dest.to_string_lossy().to_string();
|
||||
let sink = if store::inbuxa_transfer::is_s3_url(&dest_text) {
|
||||
let transfer = store::inbuxa_transfer::S3Transfer::open(&dest_text)
|
||||
.unwrap_or_else(|err| failed(&format!("Export failed: {err}")));
|
||||
Sink::S3(transfer.join(&store::inbuxa_transfer::export_name(store::write::now())))
|
||||
} else {
|
||||
Sink::Dir(params.dest.clone())
|
||||
};
|
||||
if let Sink::S3(_) = &sink {
|
||||
} else if !params.dest.exists() {
|
||||
std::fs::create_dir_all(¶ms.dest).failed("Failed to create backup directory");
|
||||
} else if !params.dest.is_dir() {
|
||||
eprintln!("Backup destination {:?} is not a directory.", params.dest);
|
||||
@@ -73,7 +95,9 @@ impl Core {
|
||||
.failed("Could not retrieve database schema version.")
|
||||
.failed("Could not retrieve database schema version.");
|
||||
|
||||
let mut families_given = true;
|
||||
if params.families.is_empty() {
|
||||
families_given = false;
|
||||
params.families = [
|
||||
Family::Data,
|
||||
Family::Registry,
|
||||
@@ -89,34 +113,81 @@ impl Core {
|
||||
.collect();
|
||||
}
|
||||
|
||||
for subspace in params
|
||||
.families
|
||||
.into_iter()
|
||||
.flat_map(|f| f.subspaces())
|
||||
.copied()
|
||||
{
|
||||
// inbuxa: K-C11: blobs last. An export taken while the server runs
|
||||
// then holds every blob the rest of it names: a message stored
|
||||
// during the export is either in it whole or not at all. (In a set's
|
||||
// order, the blobs could go first, and the messages stored after
|
||||
// them would point at blobs the export never got.)
|
||||
let mut families = params.families.into_iter().collect::<Vec<_>>();
|
||||
families.sort_by_key(|f| (*f == Family::Blob, *f as u8));
|
||||
|
||||
let mut uploads: Vec<(String, u8, Upload)> = Vec::new();
|
||||
for subspace in families.into_iter().flat_map(|f| f.subspaces()).copied() {
|
||||
let name = format!("subspace_{}", store::subspace_name(subspace));
|
||||
let (writer, upload) = match &sink {
|
||||
Sink::Dir(dir) => (Output::File(dir.join(&name)), None),
|
||||
Sink::S3(transfer) => {
|
||||
println!("Exporting database to {}/{name}.", transfer.location());
|
||||
let (writer, upload) = transfer.writer(&name);
|
||||
(Output::S3(writer), Some(upload))
|
||||
}
|
||||
};
|
||||
let (async_handle, sync_handle) = if subspace == SUBSPACE_BLOBS {
|
||||
self.backup_blobs(¶ms.dest, subspace, schema_version)
|
||||
self.backup_blobs(writer, subspace, schema_version)
|
||||
} else {
|
||||
self.backup_subspace(¶ms.dest, subspace, schema_version)
|
||||
self.backup_subspace(writer, subspace, schema_version)
|
||||
};
|
||||
async_handle.await.failed("Task failed");
|
||||
sync_handles.push(sync_handle);
|
||||
if let Some(upload) = upload {
|
||||
uploads.push((name, subspace, upload));
|
||||
}
|
||||
}
|
||||
|
||||
for handle in sync_handles {
|
||||
handle.join().expect("Failed to join thread");
|
||||
}
|
||||
|
||||
// inbuxa: K-C11: the uploads, then the manifest that says the
|
||||
// export is whole
|
||||
if let Sink::S3(transfer) = &sink {
|
||||
let mut files = Vec::new();
|
||||
let mut bytes = 0u64;
|
||||
for (name, subspace, upload) in uploads {
|
||||
let size = upload
|
||||
.await
|
||||
.failed("Upload task failed")
|
||||
.unwrap_or_else(|err| failed(&format!("Export failed: {err}")));
|
||||
bytes += size;
|
||||
files.push(serde_json::json!({"name": name, "subspace": subspace, "bytes": size}));
|
||||
}
|
||||
let manifest = serde_json::json!({
|
||||
"format": 1,
|
||||
"version": types::brand_version!(),
|
||||
"schemaVersion": schema_version,
|
||||
"createdAt": store::write::now(),
|
||||
"partial": families_given,
|
||||
"files": files,
|
||||
});
|
||||
transfer
|
||||
.put(
|
||||
S3_EXPORT_MANIFEST,
|
||||
&serde_json::to_vec_pretty(&manifest).unwrap_or_default(),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|err| failed(&format!("Export failed: {err}")));
|
||||
println!(
|
||||
"Exported {} files, {bytes} bytes compressed, to {}.",
|
||||
manifest["files"].as_array().map_or(0, Vec::len),
|
||||
transfer.location()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn backup_blobs(&self, dest: &Path, subspace: u8, schema_version: u32) -> TaskHandle {
|
||||
fn backup_blobs(&self, output: Output, subspace: u8, schema_version: u32) -> TaskHandle {
|
||||
let store = self.storage.data.clone();
|
||||
let blob_store = self.storage.blob.clone();
|
||||
let (handle, writer) = spawn_writer(
|
||||
dest.join(format!("subspace_{}", store::subspace_name(subspace))),
|
||||
subspace,
|
||||
schema_version,
|
||||
);
|
||||
let (handle, writer) = spawn_writer(output, subspace, schema_version);
|
||||
(
|
||||
tokio::spawn(async move {
|
||||
let mut blobs = Vec::new();
|
||||
@@ -175,13 +246,9 @@ impl Core {
|
||||
)
|
||||
}
|
||||
|
||||
fn backup_subspace(&self, dest: &Path, subspace: u8, schema_version: u32) -> TaskHandle {
|
||||
fn backup_subspace(&self, output: Output, subspace: u8, schema_version: u32) -> TaskHandle {
|
||||
let store = self.storage.data.clone();
|
||||
let (handle, writer) = spawn_writer(
|
||||
dest.join(format!("subspace_{}", store::subspace_name(subspace))),
|
||||
subspace,
|
||||
schema_version,
|
||||
);
|
||||
let (handle, writer) = spawn_writer(output, subspace, schema_version);
|
||||
(
|
||||
tokio::spawn(async move {
|
||||
if !store.is_sql() || (subspace != SUBSPACE_COUNTER && subspace != SUBSPACE_QUOTA) {
|
||||
@@ -253,20 +320,60 @@ impl Core {
|
||||
}
|
||||
}
|
||||
|
||||
/// inbuxa: K-C11: one export file's destination
|
||||
enum Output {
|
||||
File(PathBuf),
|
||||
S3(store::inbuxa_transfer::PartWriter),
|
||||
}
|
||||
|
||||
/// inbuxa: K-C11: the sending end of an export file's writer. The export's
|
||||
/// tasks send from the runtime's workers; when the writer is behind (an
|
||||
/// upload to S3 waits on the runtime too), a plain blocking send would hold
|
||||
/// the worker it needs, and with one worker (a pod limited to one CPU) the
|
||||
/// export would stop there. A send that has to wait hands its worker over
|
||||
/// first.
|
||||
struct ExportSender(SyncSender<(Vec<u8>, Vec<u8>)>);
|
||||
|
||||
impl ExportSender {
|
||||
fn send(&self, item: (Vec<u8>, Vec<u8>)) -> Result<(), mpsc::SendError<(Vec<u8>, Vec<u8>)>> {
|
||||
match self.0.try_send(item) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(mpsc::TrySendError::Full(item)) => {
|
||||
if can_block_in_place() {
|
||||
tokio::task::block_in_place(|| self.0.send(item))
|
||||
} else {
|
||||
self.0.send(item)
|
||||
}
|
||||
}
|
||||
Err(mpsc::TrySendError::Disconnected(item)) => Err(mpsc::SendError(item)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether this thread is a worker of a multi-threaded runtime, the only
|
||||
/// kind `block_in_place` works on
|
||||
fn can_block_in_place() -> bool {
|
||||
tokio::runtime::Handle::try_current()
|
||||
.is_ok_and(|h| h.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread)
|
||||
}
|
||||
|
||||
#[allow(clippy::type_complexity)]
|
||||
fn spawn_writer(
|
||||
path: PathBuf,
|
||||
output: Output,
|
||||
subspace: u8,
|
||||
version: u32,
|
||||
) -> (std::thread::JoinHandle<()>, SyncSender<(Vec<u8>, Vec<u8>)>) {
|
||||
) -> (std::thread::JoinHandle<()>, ExportSender) {
|
||||
let (tx, rx) = mpsc::sync_channel::<(Vec<u8>, Vec<u8>)>(10);
|
||||
|
||||
let handle = std::thread::spawn(move || {
|
||||
println!("Exporting database to {}.", path.to_str().unwrap());
|
||||
|
||||
let mut file = FrameEncoder::new(BufWriter::new(
|
||||
std::fs::File::create(path).failed("Failed to create backup file"),
|
||||
));
|
||||
let out: Box<dyn Write + Send> = match output {
|
||||
Output::File(path) => {
|
||||
println!("Exporting database to {}.", path.to_str().unwrap());
|
||||
Box::new(std::fs::File::create(path).failed("Failed to create backup file"))
|
||||
}
|
||||
Output::S3(writer) => Box::new(writer),
|
||||
};
|
||||
let mut file = FrameEncoder::new(BufWriter::new(out));
|
||||
file.write_all(&[MAGIC_MARKER, subspace])
|
||||
.failed("Failed to write version");
|
||||
file.write_all(&version.to_le_bytes())
|
||||
@@ -290,7 +397,7 @@ fn spawn_writer(
|
||||
file.flush().failed("Failed to flush backup file");
|
||||
});
|
||||
|
||||
(handle, tx)
|
||||
(handle, ExportSender(tx))
|
||||
}
|
||||
|
||||
impl BackupParams {
|
||||
|
||||
@@ -42,6 +42,9 @@ pub struct BootManager {
|
||||
/// inbuxa: K-S8: whether the store was new when this process started,
|
||||
/// before the defaults went in
|
||||
pub store_was_new: bool,
|
||||
/// inbuxa: K-C11: started with `--store-check quick|full`: like
|
||||
/// `--upgrade-plan`, then main runs a store check here and exits
|
||||
pub store_check: Option<String>,
|
||||
}
|
||||
|
||||
pub struct IpcReceivers {
|
||||
@@ -61,13 +64,14 @@ Usage: inbuxa [OPTIONS]
|
||||
|
||||
Options:
|
||||
-c, --config <PATH> Start server with the specified configuration file
|
||||
-e, --export <PATH> Export all store data to a specific path
|
||||
-i, --import <PATH> Import store data from a specific path
|
||||
-e, --export <PATH> Export all store data to a directory or s3://bucket/prefix
|
||||
-i, --import <PATH> Import store data from a directory or s3:// address
|
||||
-o, --console Open the store console
|
||||
-u, --upgrade-store Upgrade the store's layout, then exit
|
||||
--seed <PATH> Set up a new store from a file, then exit
|
||||
--upgrade-plan Print what --upgrade-store would do, then exit
|
||||
--check Check the data, blob store and coordinator, then exit
|
||||
--store-check <quick|full> Check every blob the store names, then exit
|
||||
-h, --help Print help
|
||||
-V, --version Print version
|
||||
|
||||
@@ -78,6 +82,20 @@ queues the index to be rebuilt when the server next starts. EXPORT_TYPES
|
||||
limits an export to some of: data, registry, blob, changelog, queue, report,
|
||||
telemetry, tasks.
|
||||
|
||||
An export to s3://bucket/prefix streams to a new folder there, named for the
|
||||
time (20261119T204500Z), and writes export.json last: an export without it
|
||||
didn't finish. An import from s3://bucket/prefix/<name> reads one back; from
|
||||
s3://bucket/prefix, the newest finished one. INBUXA_EXPORT_S3_ENDPOINT names
|
||||
an S3-compatible service (otherwise AWS), INBUXA_EXPORT_S3_REGION its region,
|
||||
and INBUXA_EXPORT_S3_ACCESS_KEY_FILE and INBUXA_EXPORT_S3_SECRET_KEY_FILE
|
||||
files holding its keys (otherwise the AWS_ variables or the instance's role).
|
||||
|
||||
--store-check runs a store check, as the console's Store checks page does:
|
||||
quick finds blobs that are missing, full also reads each one and checks its
|
||||
hash. It is saved with the others, and prints a summary. It exits 0 when it
|
||||
found nothing wrong, 3 when it found problems (the console lists them),
|
||||
2 when another check was running, 1 when it failed.
|
||||
|
||||
--upgrade-store does what a start does to the store before it serves, and
|
||||
exits: 0 when the store is at this release's layout (whether it upgraded it
|
||||
or found it there), 1 when the upgrade failed, 2 when it refused because
|
||||
@@ -119,6 +137,8 @@ enum StoreOp {
|
||||
UpgradePlan,
|
||||
// inbuxa: K-C5
|
||||
Check,
|
||||
// inbuxa: K-C11
|
||||
StoreCheck(String),
|
||||
None,
|
||||
}
|
||||
|
||||
@@ -150,7 +170,7 @@ impl BootManager {
|
||||
};
|
||||
let takes_value = matches!(
|
||||
key.as_str(),
|
||||
"config" | "c" | "export" | "e" | "import" | "i" | "seed"
|
||||
"config" | "c" | "export" | "e" | "import" | "i" | "seed" | "store-check"
|
||||
);
|
||||
let value = if takes_value {
|
||||
inline.or_else(|| args.next())
|
||||
@@ -191,6 +211,10 @@ impl BootManager {
|
||||
("seed", Some(value)) => {
|
||||
import_export = StoreOp::Seed(value.into());
|
||||
}
|
||||
("store-check", Some(value)) => match value.as_str() {
|
||||
"quick" | "full" => import_export = StoreOp::StoreCheck(value),
|
||||
_ => failed("--store-check takes quick or full."),
|
||||
},
|
||||
(_, None) if takes_value => failed(&format!(
|
||||
"Missing value for argument '{key}', try '--help'."
|
||||
)),
|
||||
@@ -223,7 +247,11 @@ impl BootManager {
|
||||
};
|
||||
// inbuxa: K-S8: a seed run boots as --upgrade-store does
|
||||
let upgrade_plan = import_export == StoreOp::UpgradePlan;
|
||||
let one_shot = upgrade_store || upgrade_plan || seed.is_some();
|
||||
let store_check = match &import_export {
|
||||
StoreOp::StoreCheck(mode) => Some(mode.clone()),
|
||||
_ => None,
|
||||
};
|
||||
let one_shot = upgrade_store || upgrade_plan || seed.is_some() || store_check.is_some();
|
||||
let registry = RegistryStore::init(
|
||||
PathBuf::from(config_path.unwrap()),
|
||||
import_export == StoreOp::None || one_shot,
|
||||
@@ -232,8 +260,9 @@ impl BootManager {
|
||||
.unwrap_or_else(|err| startup_failed(err));
|
||||
if one_shot && (registry.is_bootstrap_mode() || registry.is_recovery_mode()) {
|
||||
failed(
|
||||
"--upgrade-store and --seed need a configured server: there is no data \
|
||||
store in bootstrap mode, and recovery mode is for a person at the console.",
|
||||
"--upgrade-store, --seed and --store-check need a configured server: there is \
|
||||
no data store in bootstrap mode, and recovery mode is for a person at the \
|
||||
console.",
|
||||
);
|
||||
}
|
||||
let mut bootstrap = Bootstrap::new(registry).await;
|
||||
@@ -267,7 +296,11 @@ impl BootManager {
|
||||
let telemetry = Telemetry::parse(&mut bootstrap, &storage).await;
|
||||
|
||||
match import_export {
|
||||
StoreOp::None | StoreOp::UpgradeStore | StoreOp::UpgradePlan | StoreOp::Seed(_) => {
|
||||
StoreOp::None
|
||||
| StoreOp::UpgradeStore
|
||||
| StoreOp::UpgradePlan
|
||||
| StoreOp::Seed(_)
|
||||
| StoreOp::StoreCheck(_) => {
|
||||
// Parse components
|
||||
let core: Box<Core> =
|
||||
Box::new(Box::pin(Core::parse(&mut bootstrap, storage)).await);
|
||||
@@ -285,6 +318,8 @@ impl BootManager {
|
||||
"Started with --upgrade-store: upgrading the store layout, then exiting"
|
||||
} else if upgrade_plan {
|
||||
"Started with --upgrade-plan: reporting what an upgrade would do, then exiting"
|
||||
} else if store_check.is_some() {
|
||||
"Started with --store-check: checking the store's blobs, then exiting"
|
||||
} else {
|
||||
"Started with --seed: setting up a new store from a file, then exiting"
|
||||
},
|
||||
@@ -381,6 +416,7 @@ impl BootManager {
|
||||
upgrade_plan,
|
||||
seed,
|
||||
store_was_new,
|
||||
store_check,
|
||||
}
|
||||
}
|
||||
StoreOp::Export(path) => {
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use super::backup::MAGIC_MARKER;
|
||||
use super::backup::{MAGIC_MARKER, S3_EXPORT_MANIFEST};
|
||||
use crate::{Core, DATABASE_SCHEMA_VERSION};
|
||||
use lz4_flex::frame::FrameDecoder;
|
||||
use registry::{
|
||||
@@ -22,7 +22,7 @@ use std::{
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
use store::{
|
||||
BlobStore, IterateParams, SUBSPACE_BLOBS, SUBSPACE_COUNTER, SUBSPACE_INDEXES,
|
||||
BlobStore, IterateParams, SUBSPACE_BLOBS, SUBSPACE_COUNTER, SUBSPACE_INBUXA, SUBSPACE_INDEXES,
|
||||
SUBSPACE_PROPERTY, SUBSPACE_QUOTA, SUBSPACE_REGISTRY_PK, SUBSPACE_TELEMETRY_SPAN, Store,
|
||||
U32_LEN,
|
||||
write::{
|
||||
@@ -37,6 +37,12 @@ impl Core {
|
||||
/// Imports an export into an empty store and returns the subspaces it
|
||||
/// wrote. inbuxa: the caller hands them to [`Core::queue_reindex`].
|
||||
pub async fn restore(&self, src: PathBuf) -> Vec<u8> {
|
||||
// inbuxa: K-C11: an export in S3
|
||||
let src_text = src.to_string_lossy().to_string();
|
||||
if store::inbuxa_transfer::is_s3_url(&src_text) {
|
||||
return self.restore_s3(&src_text).await;
|
||||
}
|
||||
|
||||
// Backup the core
|
||||
let paths = if src.is_dir() {
|
||||
let mut paths = Vec::new();
|
||||
@@ -52,11 +58,91 @@ impl Core {
|
||||
vec![src]
|
||||
};
|
||||
|
||||
let sources = paths.into_iter().map(Source::File).collect::<Vec<_>>();
|
||||
self.restore_sources(sources).await
|
||||
}
|
||||
|
||||
/// inbuxa: K-C11: `s3://bucket/prefix/<name>`, or the newest finished
|
||||
/// export under `s3://bucket/prefix`. An export without its manifest
|
||||
/// didn't finish, and is refused.
|
||||
async fn restore_s3(&self, url: &str) -> Vec<u8> {
|
||||
use store::inbuxa_transfer::S3Transfer;
|
||||
let base =
|
||||
S3Transfer::open(url).unwrap_or_else(|err| failed(&format!("Import failed: {err}")));
|
||||
let (transfer, manifest) = match base.get(S3_EXPORT_MANIFEST).await {
|
||||
Ok(Some(manifest)) => (base, manifest),
|
||||
Ok(None) => {
|
||||
let names = base
|
||||
.children_with(S3_EXPORT_MANIFEST)
|
||||
.await
|
||||
.unwrap_or_else(|err| failed(&format!("Import failed: {err}")));
|
||||
let Some(newest) = names.last() else {
|
||||
failed(&format!(
|
||||
"Import failed: no finished export at {url} or under it (an export is \
|
||||
finished once its {S3_EXPORT_MANIFEST} is written)."
|
||||
));
|
||||
};
|
||||
let transfer = base.join(newest);
|
||||
println!(
|
||||
"Importing the newest finished export, {}.",
|
||||
transfer.location()
|
||||
);
|
||||
let manifest = transfer
|
||||
.get(S3_EXPORT_MANIFEST)
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
.unwrap_or_else(|| failed("Import failed: the export's manifest went away"));
|
||||
(transfer, manifest)
|
||||
}
|
||||
Err(err) => failed(&format!("Import failed: {err}")),
|
||||
};
|
||||
let manifest: serde_json::Value = serde_json::from_slice(&manifest)
|
||||
.unwrap_or_else(|err| failed(&format!("Import failed: {S3_EXPORT_MANIFEST}: {err}")));
|
||||
let sources = manifest["files"]
|
||||
.as_array()
|
||||
.unwrap_or_else(|| {
|
||||
failed(&format!(
|
||||
"Import failed: {S3_EXPORT_MANIFEST} lists no files"
|
||||
))
|
||||
})
|
||||
.iter()
|
||||
.map(|file| {
|
||||
let name = file["name"].as_str().unwrap_or_default();
|
||||
let subspace = file["subspace"].as_u64().and_then(|s| u8::try_from(s).ok());
|
||||
match (name, subspace) {
|
||||
(name, Some(subspace))
|
||||
if name.starts_with("subspace_") && !name.contains('/') =>
|
||||
{
|
||||
Source::S3 {
|
||||
transfer: transfer.clone(),
|
||||
name: name.to_string(),
|
||||
subspace,
|
||||
}
|
||||
}
|
||||
_ => failed(&format!(
|
||||
"Import failed: {S3_EXPORT_MANIFEST}: a bad file entry: {file}"
|
||||
)),
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
self.restore_sources(sources).await
|
||||
}
|
||||
|
||||
async fn restore_sources(&self, paths: Vec<Source>) -> Vec<u8> {
|
||||
let mut conflicts = Vec::new();
|
||||
for path in &paths {
|
||||
let subspace = KeyValueReader::new(path).subspace;
|
||||
let subspace = path.subspace();
|
||||
// inbuxa: K-C11: every start writes the fork's own subspace, this
|
||||
// import's too: where the blob store is (W), the permission
|
||||
// grants (P), the log files' setting (T). Its records are keyed
|
||||
// values, which an import sets over, never duplicates; a store in
|
||||
// use still has data in the other subspaces, which are checked.
|
||||
if subspace == SUBSPACE_INBUXA {
|
||||
continue;
|
||||
}
|
||||
if subspace_has_data(&self.storage.data, subspace).await {
|
||||
conflicts.push(path.clone());
|
||||
conflicts.push(path.describe());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -68,34 +154,85 @@ impl Core {
|
||||
start the server before importing. Conflicting dumps:"
|
||||
);
|
||||
for path in conflicts {
|
||||
eprintln!(" {}", path.display());
|
||||
eprintln!(" {path}");
|
||||
}
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
let mut imported = paths
|
||||
.iter()
|
||||
.map(|path| KeyValueReader::new(path).subspace)
|
||||
.collect::<Vec<_>>();
|
||||
let mut imported = paths.iter().map(Source::subspace).collect::<Vec<_>>();
|
||||
imported.sort_unstable();
|
||||
imported.dedup();
|
||||
|
||||
// inbuxa: K-C11: the blobs last, into the blob store the imported
|
||||
// settings name. This process started on an empty store, whose
|
||||
// blob store is the data store; a server started on the imported
|
||||
// one reads blobs from wherever its x:BlobStore says (S3, say), so
|
||||
// blobs written to the data store would be missing there.
|
||||
let (blobs, rest): (Vec<_>, Vec<_>) = paths
|
||||
.into_iter()
|
||||
.partition(|path| path.subspace() == SUBSPACE_BLOBS);
|
||||
|
||||
let mut tasks = Vec::new();
|
||||
for path in paths {
|
||||
for path in rest {
|
||||
let storage = self.storage.clone();
|
||||
let blob_store = self.storage.blob.clone();
|
||||
tasks.push(tokio::spawn(async move {
|
||||
restore_file(storage.data, blob_store, &path).await;
|
||||
}));
|
||||
}
|
||||
|
||||
for task in tasks {
|
||||
task.await.failed("Failed to wait for task");
|
||||
}
|
||||
|
||||
if !blobs.is_empty() {
|
||||
let blob_store = self.imported_blob_store().await;
|
||||
let mut tasks = Vec::new();
|
||||
for path in blobs {
|
||||
let storage = self.storage.clone();
|
||||
let blob_store = blob_store.clone();
|
||||
tasks.push(tokio::spawn(async move {
|
||||
restore_file(storage.data, blob_store, &path).await;
|
||||
}));
|
||||
}
|
||||
for task in tasks {
|
||||
task.await.failed("Failed to wait for task");
|
||||
}
|
||||
}
|
||||
|
||||
imported
|
||||
}
|
||||
|
||||
/// inbuxa: K-C11: the blob store the store's x:BlobStore names now,
|
||||
/// after the import wrote it
|
||||
async fn imported_blob_store(&self) -> BlobStore {
|
||||
use registry::{schema::structs::BlobStore as BlobStoreSetting, types::ObjectImpl};
|
||||
let setting = self
|
||||
.storage
|
||||
.registry
|
||||
.object::<BlobStoreSetting>(BlobStoreSetting::OBJECT.singleton().id())
|
||||
.await
|
||||
.failed("Failed to read the imported blob store setting")
|
||||
.unwrap_or_default();
|
||||
if matches!(setting, BlobStoreSetting::Default) {
|
||||
println!("Importing blobs into the data store, as the export's settings say.");
|
||||
return self.storage.data.clone().into();
|
||||
}
|
||||
let mut warnings = Vec::new();
|
||||
let store = BlobStore::open(setting, &self.storage.data, &mut warnings)
|
||||
.await
|
||||
.unwrap_or_else(|err| {
|
||||
failed(&format!(
|
||||
"Can't open the blob store the export's settings name, to import the \
|
||||
blobs into: {err}"
|
||||
))
|
||||
});
|
||||
for warning in warnings {
|
||||
eprintln!("Blob store: {warning}");
|
||||
}
|
||||
println!("Importing blobs into the blob store the export's settings name.");
|
||||
store
|
||||
}
|
||||
|
||||
/// inbuxa: an export never carries the full-text index. It is built by
|
||||
/// and for one search backend (the SQL stores index into their own
|
||||
/// tables, the key-value stores into a subspace, external engines keep it
|
||||
@@ -172,10 +309,59 @@ async fn subspace_has_data(store: &Store, subspace: u8) -> bool {
|
||||
has_data
|
||||
}
|
||||
|
||||
async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) {
|
||||
println!("Importing database dump from {}.", path.to_str().unwrap());
|
||||
/// inbuxa: K-C11: one export file: on disk, or in S3
|
||||
enum Source {
|
||||
File(PathBuf),
|
||||
S3 {
|
||||
transfer: store::inbuxa_transfer::S3Transfer,
|
||||
name: String,
|
||||
/// From the manifest, so the file needn't be opened to know it
|
||||
subspace: u8,
|
||||
},
|
||||
}
|
||||
|
||||
let mut reader = KeyValueReader::new(path);
|
||||
impl Source {
|
||||
fn subspace(&self) -> u8 {
|
||||
match self {
|
||||
Source::File(path) => KeyValueReader::new(path).subspace,
|
||||
Source::S3 { subspace, .. } => *subspace,
|
||||
}
|
||||
}
|
||||
|
||||
fn describe(&self) -> String {
|
||||
match self {
|
||||
Source::File(path) => path.display().to_string(),
|
||||
Source::S3 { transfer, name, .. } => format!("{}/{name}", transfer.location()),
|
||||
}
|
||||
}
|
||||
|
||||
fn open(&self) -> KeyValueReader {
|
||||
match self {
|
||||
Source::File(path) => KeyValueReader::new(path),
|
||||
Source::S3 {
|
||||
transfer,
|
||||
name,
|
||||
subspace,
|
||||
} => {
|
||||
let reader =
|
||||
KeyValueReader::from_reader(Box::new(transfer.reader(name)), &self.describe());
|
||||
if reader.subspace != *subspace {
|
||||
failed(&format!(
|
||||
"{} holds subspace {}, but the manifest says {subspace}",
|
||||
self.describe(),
|
||||
reader.subspace
|
||||
));
|
||||
}
|
||||
reader
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn restore_file(store: Store, blob_store: BlobStore, path: &Source) {
|
||||
println!("Importing database dump from {}.", path.describe());
|
||||
|
||||
let mut reader = path.open();
|
||||
let mut batch = BatchBuilder::new();
|
||||
|
||||
match reader.subspace {
|
||||
@@ -276,14 +462,20 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) {
|
||||
|
||||
struct KeyValueReader {
|
||||
subspace: u8,
|
||||
file: FrameDecoder<BufReader<File>>,
|
||||
file: FrameDecoder<BufReader<Box<dyn Read + Send>>>,
|
||||
}
|
||||
|
||||
impl KeyValueReader {
|
||||
fn new(path: &Path) -> Self {
|
||||
let mut file = FrameDecoder::new(BufReader::new(
|
||||
File::open(path).failed("Failed to open file"),
|
||||
));
|
||||
Self::from_reader(
|
||||
Box::new(File::open(path).failed("Failed to open file")),
|
||||
&path.display().to_string(),
|
||||
)
|
||||
}
|
||||
|
||||
// inbuxa: K-C11: any reader: a file, or a download from S3
|
||||
fn from_reader(reader: Box<dyn Read + Send>, path: &str) -> Self {
|
||||
let mut file = FrameDecoder::new(BufReader::new(reader));
|
||||
let mut buf = [0u8; 1];
|
||||
file.read_exact(&mut buf)
|
||||
.failed(&format!("Failed to read magic marker from {path:?}"));
|
||||
|
||||
@@ -80,6 +80,12 @@ async fn main() -> std::io::Result<()> {
|
||||
std::process::exit(Box::pin(seed_and_exit(&server, &path, init.store_was_new)).await);
|
||||
}
|
||||
|
||||
// inbuxa: K-C11: --store-check runs a check and exits
|
||||
if let Some(mode) = init.store_check.clone() {
|
||||
let server = init.inner.build_server();
|
||||
std::process::exit(Box::pin(store_check_and_exit(&server, &mode)).await);
|
||||
}
|
||||
|
||||
// Migrate database
|
||||
if let Err(err) = migration::try_migrate(&init.inner.build_server()).await {
|
||||
trc::event!(
|
||||
@@ -274,6 +280,60 @@ async fn seed_and_exit(
|
||||
flush_and_exit_code(code).await
|
||||
}
|
||||
|
||||
/// inbuxa: K-C11: `--store-check`: 0 found nothing wrong, 3 found problems,
|
||||
/// 2 another check was running, 1 failed
|
||||
async fn store_check_and_exit(server: &common::Server, mode: &str) -> i32 {
|
||||
use services::inbuxa_store_check::{
|
||||
CheckMode as Mode, CheckRefusal as Refusal, CheckStatus as Status,
|
||||
};
|
||||
|
||||
let mode = Mode::parse(mode).unwrap_or(Mode::Quick);
|
||||
let (code, message) = match services::inbuxa_store_check::run_now(server, mode).await {
|
||||
Ok(Ok(check)) => {
|
||||
let p = &check.progress;
|
||||
let summary = format!(
|
||||
"Store check {} ({}): {} blobs, {} links, {} messages checked, {} bytes read. \
|
||||
Missing {}, damaged {}, unlinked {}, unreadable {}.",
|
||||
check.id,
|
||||
mode.as_str(),
|
||||
p.blobs_checked,
|
||||
p.links_checked,
|
||||
p.messages_checked,
|
||||
p.bytes_read,
|
||||
p.missing,
|
||||
p.damaged,
|
||||
p.unlinked,
|
||||
p.unreadable,
|
||||
);
|
||||
match check.status {
|
||||
Status::Done if p.problems() == 0 => (0, format!("{summary} Nothing wrong.")),
|
||||
Status::Done => (
|
||||
3,
|
||||
format!(
|
||||
"{summary} {} problems: the console's Store checks page lists them.",
|
||||
p.problems()
|
||||
),
|
||||
),
|
||||
status => (
|
||||
1,
|
||||
format!(
|
||||
"{summary} It ended {}: {}",
|
||||
status.as_str(),
|
||||
check.error.as_deref().unwrap_or("no reason given")
|
||||
),
|
||||
),
|
||||
}
|
||||
}
|
||||
Ok(Err(Refusal::Running(id))) => (
|
||||
2,
|
||||
format!("Store check {id} is running: one check runs at a time."),
|
||||
),
|
||||
Err(err) => (1, format!("The store check failed: {err}")),
|
||||
};
|
||||
println!("{message}");
|
||||
flush_and_exit_code(code).await
|
||||
}
|
||||
|
||||
/// inbuxa: K-C9: `--upgrade-plan`: one line of JSON on stdout, for a
|
||||
/// script to read, and the exit code a shell can branch on
|
||||
async fn upgrade_plan_and_exit(server: &common::Server) -> i32 {
|
||||
|
||||
@@ -74,6 +74,35 @@ pub async fn start(
|
||||
Ok(Ok(id))
|
||||
}
|
||||
|
||||
// inbuxa: K-C11: for main's --store-check
|
||||
pub use inbuxa_features::store_check::{
|
||||
Mode as CheckMode, Refusal as CheckRefusal, Status as CheckStatus,
|
||||
};
|
||||
|
||||
/// inbuxa: K-C11: the node id a `--store-check` run records. Leased node
|
||||
/// ids are 16-bit, so no server's start-up (SV-8) takes it for its own.
|
||||
pub const ONE_SHOT_NODE_ID: u64 = 1 << 32;
|
||||
|
||||
/// inbuxa: K-C11: `--store-check`: a check run here, to its end, for a
|
||||
/// scheduled job. Saved like any other, so the console shows it.
|
||||
pub async fn run_now(server: &Server, mode: Mode) -> trc::Result<Result<StoreCheck, Refusal>> {
|
||||
let now = now();
|
||||
let check = StoreCheck::new(mode, "store-check job".to_string(), ONE_SHOT_NODE_ID, now);
|
||||
let id = match model::create(server.store(), &check, now).await? {
|
||||
Ok(id) => id,
|
||||
Err(refusal) => return Ok(Err(refusal)),
|
||||
};
|
||||
run(server, StoreCheck { id, ..check }).await;
|
||||
Ok(model::get(server.store(), id)
|
||||
.await?
|
||||
.ok_or_else(|| {
|
||||
trc::StoreEvent::NotFound
|
||||
.into_err()
|
||||
.details("The check went away")
|
||||
})
|
||||
.map(Ok)?)
|
||||
}
|
||||
|
||||
/// SV-8: asks a running check to stop. Any node can: the one running it
|
||||
/// sees the request at its next save. None if there's no such check.
|
||||
pub async fn stop(server: &Server, id: u32) -> trc::Result<Option<StoreCheck>> {
|
||||
|
||||
@@ -65,7 +65,7 @@ foundation = ["foundationdb", "futures"]
|
||||
fdb-chunked-bm = []
|
||||
|
||||
# Blob stores
|
||||
s3 = ["rust-s3", "dep:reqwest_s3"]
|
||||
s3 = ["rust-s3", "dep:reqwest_s3", "futures"]
|
||||
azure = ["azure_core", "azure_storage", "azure_storage_blobs", "futures"]
|
||||
|
||||
# In-memory stores
|
||||
|
||||
@@ -0,0 +1,592 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
//! inbuxa: K-C11: an export written to, and read back from, an S3 bucket.
|
||||
//!
|
||||
//! `--export s3://bucket/prefix` streams each export file into a multipart
|
||||
//! upload, so nothing the size of the store lands on local disk, and
|
||||
//! `--import s3://bucket/prefix/<name>` streams it back. The bucket's
|
||||
//! address and keys come from the environment:
|
||||
//!
|
||||
//! - `INBUXA_EXPORT_S3_ENDPOINT`: an S3-compatible service (MinIO, Garage,
|
||||
//! Ceph...), addressed path-style. Without it, AWS.
|
||||
//! - `INBUXA_EXPORT_S3_REGION`: `us-east-1` unless given.
|
||||
//! - `INBUXA_EXPORT_S3_ACCESS_KEY_FILE`, `INBUXA_EXPORT_S3_SECRET_KEY_FILE`:
|
||||
//! files holding the keys. Without them, the usual AWS sources (the
|
||||
//! `AWS_` variables, a profile, the instance's role).
|
||||
//!
|
||||
//! The export and import code is sync (a writer thread per file, readers in
|
||||
//! the import tasks); these bridge it to the async S3 client.
|
||||
|
||||
/// Whether a `--export`/`--import` value names a bucket.
|
||||
pub fn is_s3_url(value: &str) -> bool {
|
||||
value.starts_with("s3://")
|
||||
}
|
||||
|
||||
/// The name an export made now goes under: `20261119T204500Z`.
|
||||
pub fn export_name(now: u64) -> String {
|
||||
let days = (now / 86_400) as i64;
|
||||
let secs = now % 86_400;
|
||||
// Howard Hinnant's civil_from_days
|
||||
let z = days + 719_468;
|
||||
let era = z.div_euclid(146_097);
|
||||
let doe = z.rem_euclid(146_097);
|
||||
let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
|
||||
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
|
||||
let mp = (5 * doy + 2) / 153;
|
||||
let day = doy - (153 * mp + 2) / 5 + 1;
|
||||
let month = if mp < 10 { mp + 3 } else { mp - 9 };
|
||||
let year = yoe + era * 400 + i64::from(month <= 2);
|
||||
format!(
|
||||
"{year:04}{month:02}{day:02}T{:02}{:02}{:02}Z",
|
||||
secs / 3600,
|
||||
(secs / 60) % 60,
|
||||
secs % 60
|
||||
)
|
||||
}
|
||||
|
||||
/// The size of upload part `n` (from 1): 16 MiB for the first thousand,
|
||||
/// doubling every thousand after, up to 512 MiB. S3 takes at most 10,000
|
||||
/// parts, so one file can be nearly 3 TiB, and only one part is held at a
|
||||
/// time.
|
||||
pub fn part_size(part: u32) -> usize {
|
||||
const MIB: usize = 1024 * 1024;
|
||||
let tier = (part.saturating_sub(1) / 1000).min(5);
|
||||
(16 * MIB) << tier
|
||||
}
|
||||
|
||||
/// The most parts S3 takes in one upload.
|
||||
pub const MAX_PARTS: u32 = 10_000;
|
||||
|
||||
#[cfg(feature = "s3")]
|
||||
pub use imp::{PartReader, PartWriter, S3Transfer};
|
||||
|
||||
#[cfg(not(feature = "s3"))]
|
||||
pub use none::{PartReader, PartWriter, S3Transfer};
|
||||
|
||||
#[cfg(feature = "s3")]
|
||||
mod imp {
|
||||
use super::{MAX_PARTS, part_size};
|
||||
use futures::StreamExt;
|
||||
use s3::{Bucket, Region, creds::Credentials, serde_types::Part};
|
||||
use std::io::{Read, Write};
|
||||
use std::sync::mpsc as std_mpsc;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
const CONTENT_TYPE: &str = "application/octet-stream";
|
||||
const RETRIES: u32 = 5;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct S3Transfer {
|
||||
bucket: Box<Bucket>,
|
||||
bucket_name: String,
|
||||
/// No leading or trailing slash; may be empty.
|
||||
prefix: String,
|
||||
}
|
||||
|
||||
fn env(name: &str) -> Option<String> {
|
||||
std::env::var(name)
|
||||
.ok()
|
||||
.map(|v| v.trim().to_string())
|
||||
.filter(|v| !v.is_empty())
|
||||
}
|
||||
|
||||
fn key_file(name: &str) -> Result<Option<String>, String> {
|
||||
match env(name) {
|
||||
Some(path) => std::fs::read_to_string(&path)
|
||||
.map(|v| Some(v.trim_end_matches(['\r', '\n']).to_string()))
|
||||
.map_err(|err| format!("{name}: can't read {path}: {err}")),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
impl S3Transfer {
|
||||
/// `s3://bucket/prefix`, with the bucket's address and keys from the
|
||||
/// environment.
|
||||
pub fn open(url: &str) -> Result<Self, String> {
|
||||
let rest = url
|
||||
.strip_prefix("s3://")
|
||||
.ok_or_else(|| format!("{url}: not an s3:// address"))?;
|
||||
let (bucket_name, prefix) = rest.split_once('/').unwrap_or((rest, ""));
|
||||
if bucket_name.is_empty() {
|
||||
return Err(format!("{url}: no bucket named"));
|
||||
}
|
||||
let region_name = env("INBUXA_EXPORT_S3_REGION").unwrap_or_else(|| "us-east-1".into());
|
||||
let endpoint = env("INBUXA_EXPORT_S3_ENDPOINT");
|
||||
let region = match &endpoint {
|
||||
Some(endpoint) => Region::Custom {
|
||||
region: region_name,
|
||||
endpoint: endpoint.trim_end_matches('/').to_string(),
|
||||
},
|
||||
None => region_name
|
||||
.parse()
|
||||
.map_err(|err| format!("INBUXA_EXPORT_S3_REGION: {err}"))?,
|
||||
};
|
||||
let access = key_file("INBUXA_EXPORT_S3_ACCESS_KEY_FILE")?;
|
||||
let secret = key_file("INBUXA_EXPORT_S3_SECRET_KEY_FILE")?;
|
||||
let credentials = match (&access, &secret) {
|
||||
(Some(access), Some(secret)) => {
|
||||
Credentials::new(Some(access), Some(secret), None, None, None)
|
||||
}
|
||||
(None, None) => Credentials::default(),
|
||||
_ => {
|
||||
return Err("INBUXA_EXPORT_S3_ACCESS_KEY_FILE and \
|
||||
INBUXA_EXPORT_S3_SECRET_KEY_FILE go together"
|
||||
.into());
|
||||
}
|
||||
}
|
||||
.map_err(|err| format!("S3 credentials: {err}"))?;
|
||||
let mut bucket = Bucket::new(bucket_name, region, credentials)
|
||||
.map_err(|err| format!("S3 bucket {bucket_name}: {err}"))?;
|
||||
if endpoint.is_some() {
|
||||
bucket = bucket.with_path_style();
|
||||
}
|
||||
Ok(S3Transfer {
|
||||
bucket,
|
||||
bucket_name: bucket_name.to_string(),
|
||||
prefix: prefix.trim_matches('/').to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
/// The same bucket, one level down.
|
||||
pub fn join(&self, name: &str) -> Self {
|
||||
S3Transfer {
|
||||
prefix: self.key(name),
|
||||
..self.clone()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn location(&self) -> String {
|
||||
format!("s3://{}/{}", self.bucket_name, self.prefix)
|
||||
}
|
||||
|
||||
fn key(&self, name: &str) -> String {
|
||||
if self.prefix.is_empty() {
|
||||
name.to_string()
|
||||
} else {
|
||||
format!("{}/{name}", self.prefix)
|
||||
}
|
||||
}
|
||||
|
||||
/// A small object, whole.
|
||||
pub async fn put(&self, name: &str, data: &[u8]) -> Result<(), String> {
|
||||
let key = self.key(name);
|
||||
let mut last = String::new();
|
||||
for attempt in 0..RETRIES {
|
||||
match self.bucket.put_object(&key, data).await {
|
||||
Ok(r) if (200..300).contains(&r.status_code()) => return Ok(()),
|
||||
Ok(r) => {
|
||||
last = format!(
|
||||
"{}: {}",
|
||||
r.status_code(),
|
||||
String::from_utf8_lossy(r.as_slice())
|
||||
)
|
||||
}
|
||||
Err(err) => last = err.to_string(),
|
||||
}
|
||||
wait(attempt).await;
|
||||
}
|
||||
Err(format!(
|
||||
"Failed to write {}: {last}",
|
||||
self.location_of(name)
|
||||
))
|
||||
}
|
||||
|
||||
/// A small object, whole; None when it isn't there.
|
||||
pub async fn get(&self, name: &str) -> Result<Option<Vec<u8>>, String> {
|
||||
let key = self.key(name);
|
||||
let mut last = String::new();
|
||||
for attempt in 0..RETRIES {
|
||||
match self.bucket.get_object(&key).await {
|
||||
Ok(r) if (200..300).contains(&r.status_code()) => {
|
||||
return Ok(Some(r.to_vec()));
|
||||
}
|
||||
Ok(r) if r.status_code() == 404 => return Ok(None),
|
||||
Ok(r) => last = format!("{}", r.status_code()),
|
||||
Err(s3::error::S3Error::HttpFailWithBody(404, _)) => return Ok(None),
|
||||
Err(err) => last = err.to_string(),
|
||||
}
|
||||
wait(attempt).await;
|
||||
}
|
||||
Err(format!("Failed to read {}: {last}", self.location_of(name)))
|
||||
}
|
||||
|
||||
/// The names one level down that hold `marker`: the exports under a
|
||||
/// prefix.
|
||||
pub async fn children_with(&self, marker: &str) -> Result<Vec<String>, String> {
|
||||
let prefix = if self.prefix.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("{}/", self.prefix)
|
||||
};
|
||||
let results = self
|
||||
.bucket
|
||||
.list(prefix.clone(), None)
|
||||
.await
|
||||
.map_err(|err| format!("Failed to list {}: {err}", self.location()))?;
|
||||
let mut names = results
|
||||
.iter()
|
||||
.flat_map(|r| r.contents.iter())
|
||||
.filter_map(|o| o.key.strip_prefix(&prefix))
|
||||
.filter_map(|rest| rest.strip_suffix(marker)?.strip_suffix('/'))
|
||||
.filter(|name| !name.is_empty() && !name.contains('/'))
|
||||
.map(str::to_string)
|
||||
.collect::<Vec<_>>();
|
||||
names.sort();
|
||||
Ok(names)
|
||||
}
|
||||
|
||||
fn location_of(&self, name: &str) -> String {
|
||||
format!("s3://{}/{}", self.bucket_name, self.key(name))
|
||||
}
|
||||
|
||||
/// A writer whose bytes go up as a multipart upload under `name`,
|
||||
/// for a thread outside the runtime; the upload runs on the
|
||||
/// runtime. Dropping the writer sends the last part; the handle
|
||||
/// ends when the upload is complete.
|
||||
pub fn writer(
|
||||
&self,
|
||||
name: &str,
|
||||
) -> (PartWriter, tokio::task::JoinHandle<Result<u64, String>>) {
|
||||
let (tx, rx) = mpsc::channel::<Vec<u8>>(1);
|
||||
let this = self.clone();
|
||||
let name = name.to_string();
|
||||
let handle = tokio::spawn(async move { this.upload(&name, rx).await });
|
||||
(
|
||||
PartWriter {
|
||||
tx: Some(tx),
|
||||
buf: Vec::with_capacity(part_size(1)),
|
||||
part: 1,
|
||||
},
|
||||
handle,
|
||||
)
|
||||
}
|
||||
|
||||
async fn upload(&self, name: &str, mut rx: mpsc::Receiver<Vec<u8>>) -> Result<u64, String> {
|
||||
let key = self.key(name);
|
||||
let location = self.location_of(name);
|
||||
let upload_id = self
|
||||
.bucket
|
||||
.initiate_multipart_upload(&key, CONTENT_TYPE)
|
||||
.await
|
||||
.map_err(|err| format!("Failed to start uploading {location}: {err}"))?
|
||||
.upload_id;
|
||||
let mut parts: Vec<Part> = Vec::new();
|
||||
let mut total = 0u64;
|
||||
let result: Result<(), String> = async {
|
||||
while let Some(chunk) = rx.recv().await {
|
||||
let number = parts.len() as u32 + 1;
|
||||
if number > MAX_PARTS {
|
||||
return Err(format!("{location}: too large for one upload"));
|
||||
}
|
||||
total += chunk.len() as u64;
|
||||
parts.push(
|
||||
self.put_part(chunk, &key, number, &upload_id, &location)
|
||||
.await?,
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
if let Err(err) = result {
|
||||
// Dropping rx fails the writer's next send, which stops it
|
||||
drop(rx);
|
||||
let _ = self.bucket.abort_upload(&key, &upload_id).await;
|
||||
return Err(err);
|
||||
}
|
||||
let mut last = String::new();
|
||||
for attempt in 0..RETRIES {
|
||||
match self
|
||||
.bucket
|
||||
.complete_multipart_upload(&key, &upload_id, parts.clone())
|
||||
.await
|
||||
{
|
||||
Ok(r) if (200..300).contains(&r.status_code()) => return Ok(total),
|
||||
Ok(r) => last = format!("{}", r.status_code()),
|
||||
Err(err) => last = err.to_string(),
|
||||
}
|
||||
wait(attempt).await;
|
||||
}
|
||||
let _ = self.bucket.abort_upload(&key, &upload_id).await;
|
||||
Err(format!("Failed to finish uploading {location}: {last}"))
|
||||
}
|
||||
|
||||
async fn put_part(
|
||||
&self,
|
||||
chunk: Vec<u8>,
|
||||
key: &str,
|
||||
number: u32,
|
||||
upload_id: &str,
|
||||
location: &str,
|
||||
) -> Result<Part, String> {
|
||||
let mut last = String::new();
|
||||
for attempt in 0..RETRIES {
|
||||
match self
|
||||
.bucket
|
||||
.put_multipart_chunk(chunk.clone(), key, number, upload_id, CONTENT_TYPE)
|
||||
.await
|
||||
{
|
||||
Ok(part) => return Ok(part),
|
||||
Err(err) => last = err.to_string(),
|
||||
}
|
||||
wait(attempt).await;
|
||||
}
|
||||
Err(format!(
|
||||
"Failed to upload part {number} of {location}: {last}"
|
||||
))
|
||||
}
|
||||
|
||||
/// A reader of the object `name`, for code that reads synchronously
|
||||
/// on the runtime's workers: the download is a task on the runtime,
|
||||
/// and a reader that has to wait for it hands its worker over.
|
||||
pub fn reader(&self, name: &str) -> PartReader {
|
||||
let (tx, rx) = std_mpsc::sync_channel::<Result<Vec<u8>, String>>(4);
|
||||
let this = self.clone();
|
||||
let key = self.key(name);
|
||||
let location = self.location_of(name);
|
||||
tokio::spawn(async move {
|
||||
{
|
||||
let mut stream = match this.bucket.get_object_stream(&key).await {
|
||||
Ok(stream) if (200..300).contains(&stream.status_code) => stream,
|
||||
Ok(stream) => {
|
||||
let _ = send_chunk(
|
||||
&tx,
|
||||
Err(format!("Failed to read {location}: {}", stream.status_code)),
|
||||
);
|
||||
return;
|
||||
}
|
||||
Err(err) => {
|
||||
let _ =
|
||||
send_chunk(&tx, Err(format!("Failed to read {location}: {err}")));
|
||||
return;
|
||||
}
|
||||
};
|
||||
while let Some(chunk) = stream.bytes().next().await {
|
||||
let sent = match chunk {
|
||||
Ok(bytes) => send_chunk(&tx, Ok(bytes.to_vec())),
|
||||
Err(err) => {
|
||||
let _ = send_chunk(
|
||||
&tx,
|
||||
Err(format!("Failed to read {location}: {err}")),
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
if sent.is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
PartReader {
|
||||
rx,
|
||||
buf: Vec::new(),
|
||||
pos: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether this thread is a worker of a multi-threaded runtime, where a
|
||||
/// wait can hand the worker over (`block_in_place`)
|
||||
fn on_worker() -> bool {
|
||||
tokio::runtime::Handle::try_current()
|
||||
.is_ok_and(|h| h.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread)
|
||||
}
|
||||
|
||||
/// A send that, when it has to wait for the reader, hands the worker over
|
||||
fn send_chunk(
|
||||
tx: &std_mpsc::SyncSender<Result<Vec<u8>, String>>,
|
||||
item: Result<Vec<u8>, String>,
|
||||
) -> Result<(), ()> {
|
||||
match tx.try_send(item) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(std_mpsc::TrySendError::Disconnected(_)) => Err(()),
|
||||
Err(std_mpsc::TrySendError::Full(item)) if on_worker() => {
|
||||
tokio::task::block_in_place(|| tx.send(item)).map_err(|_| ())
|
||||
}
|
||||
Err(std_mpsc::TrySendError::Full(item)) => tx.send(item).map_err(|_| ()),
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait(attempt: u32) {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(250 << attempt.min(5))).await;
|
||||
}
|
||||
|
||||
/// See [`S3Transfer::writer`].
|
||||
pub struct PartWriter {
|
||||
tx: Option<mpsc::Sender<Vec<u8>>>,
|
||||
buf: Vec<u8>,
|
||||
part: u32,
|
||||
}
|
||||
|
||||
impl PartWriter {
|
||||
fn send(&mut self) -> std::io::Result<()> {
|
||||
let next = part_size(self.part + 1);
|
||||
let chunk = std::mem::replace(&mut self.buf, Vec::with_capacity(next));
|
||||
self.part += 1;
|
||||
match &self.tx {
|
||||
Some(tx) => tx.blocking_send(chunk).map_err(|_| {
|
||||
std::io::Error::other("the upload stopped: its error is on the next line")
|
||||
}),
|
||||
None => Err(std::io::Error::other("the upload is finished")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for PartWriter {
|
||||
fn write(&mut self, data: &[u8]) -> std::io::Result<usize> {
|
||||
let room = part_size(self.part) - self.buf.len();
|
||||
let take = room.min(data.len());
|
||||
self.buf.extend_from_slice(&data[..take]);
|
||||
if self.buf.len() == part_size(self.part) {
|
||||
self.send()?;
|
||||
}
|
||||
Ok(take)
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PartWriter {
|
||||
fn drop(&mut self) {
|
||||
// The last part, which may be short; even an empty file is one
|
||||
// part
|
||||
if !self.buf.is_empty() || self.part == 1 {
|
||||
let _ = self.send();
|
||||
}
|
||||
self.tx = None;
|
||||
}
|
||||
}
|
||||
|
||||
/// See [`S3Transfer::reader`].
|
||||
pub struct PartReader {
|
||||
rx: std_mpsc::Receiver<Result<Vec<u8>, String>>,
|
||||
buf: Vec<u8>,
|
||||
pos: usize,
|
||||
}
|
||||
|
||||
impl Read for PartReader {
|
||||
fn read(&mut self, out: &mut [u8]) -> std::io::Result<usize> {
|
||||
while self.pos == self.buf.len() {
|
||||
// Readers run on the runtime's workers (the import's tasks),
|
||||
// and the download needs one to make progress: a wait hands
|
||||
// this worker over first, or a pod with one CPU's worth of
|
||||
// workers would stop here
|
||||
let next = match self.rx.try_recv() {
|
||||
Ok(next) => Ok(next),
|
||||
Err(std_mpsc::TryRecvError::Disconnected) => Err(std_mpsc::RecvError),
|
||||
Err(std_mpsc::TryRecvError::Empty) => {
|
||||
if on_worker() {
|
||||
tokio::task::block_in_place(|| self.rx.recv())
|
||||
} else {
|
||||
self.rx.recv()
|
||||
}
|
||||
}
|
||||
};
|
||||
match next {
|
||||
Ok(Ok(chunk)) => {
|
||||
self.buf = chunk;
|
||||
self.pos = 0;
|
||||
}
|
||||
Ok(Err(err)) => return Err(std::io::Error::other(err)),
|
||||
// The download ended
|
||||
Err(_) => return Ok(0),
|
||||
}
|
||||
}
|
||||
let n = out.len().min(self.buf.len() - self.pos);
|
||||
out[..n].copy_from_slice(&self.buf[self.pos..self.pos + n]);
|
||||
self.pos += n;
|
||||
Ok(n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "s3"))]
|
||||
mod none {
|
||||
//! This build has no S3 client: [`S3Transfer::open`] always refuses, so
|
||||
//! nothing else here is ever reached.
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct S3Transfer;
|
||||
pub struct PartWriter;
|
||||
pub struct PartReader;
|
||||
|
||||
impl S3Transfer {
|
||||
pub fn open(_url: &str) -> Result<Self, String> {
|
||||
Err("this build has no S3 support (the s3 feature)".into())
|
||||
}
|
||||
pub fn join(&self, _name: &str) -> Self {
|
||||
unreachable!()
|
||||
}
|
||||
pub fn location(&self) -> String {
|
||||
unreachable!()
|
||||
}
|
||||
pub async fn put(&self, _name: &str, _data: &[u8]) -> Result<(), String> {
|
||||
unreachable!()
|
||||
}
|
||||
pub async fn get(&self, _name: &str) -> Result<Option<Vec<u8>>, String> {
|
||||
unreachable!()
|
||||
}
|
||||
pub async fn children_with(&self, _marker: &str) -> Result<Vec<String>, String> {
|
||||
unreachable!()
|
||||
}
|
||||
pub fn writer(
|
||||
&self,
|
||||
_name: &str,
|
||||
) -> (PartWriter, tokio::task::JoinHandle<Result<u64, String>>) {
|
||||
unreachable!()
|
||||
}
|
||||
pub fn reader(&self, _name: &str) -> PartReader {
|
||||
unreachable!()
|
||||
}
|
||||
}
|
||||
|
||||
impl std::io::Write for PartWriter {
|
||||
fn write(&mut self, _data: &[u8]) -> std::io::Result<usize> {
|
||||
unreachable!()
|
||||
}
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
unreachable!()
|
||||
}
|
||||
}
|
||||
|
||||
impl std::io::Read for PartReader {
|
||||
fn read(&mut self, _out: &mut [u8]) -> std::io::Result<usize> {
|
||||
unreachable!()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn export_names_sort_by_time() {
|
||||
assert_eq!(export_name(0), "19700101T000000Z");
|
||||
assert_eq!(export_name(1_795_552_496), "20261124T203456Z");
|
||||
assert_eq!(export_name(951_782_400), "20000229T000000Z");
|
||||
assert!(export_name(1_795_552_496) < export_name(1_795_552_497));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parts_grow_to_nearly_three_tebibytes() {
|
||||
assert_eq!(part_size(1), 16 << 20);
|
||||
assert_eq!(part_size(1000), 16 << 20);
|
||||
assert_eq!(part_size(1001), 32 << 20);
|
||||
assert_eq!(part_size(MAX_PARTS), 512 << 20);
|
||||
let total: u64 = (1..=MAX_PARTS).map(|n| part_size(n) as u64).sum();
|
||||
assert!(total > (29u64 << 40) / 10, "{total}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn s3_urls() {
|
||||
assert!(is_s3_url("s3://backups/inbuxa"));
|
||||
assert!(!is_s3_url("/var/backups"));
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,8 @@
|
||||
pub mod backend;
|
||||
pub mod build;
|
||||
pub mod dispatch;
|
||||
// inbuxa: K-C11: exports to and from S3
|
||||
pub mod inbuxa_transfer;
|
||||
pub mod query;
|
||||
pub mod registry;
|
||||
pub mod search;
|
||||
|
||||
@@ -278,6 +278,31 @@ pub async fn test(test: &TestServer) {
|
||||
.backup(BackupParams::new(temp_dir.path.clone()))
|
||||
.await;
|
||||
|
||||
// inbuxa: K-C11: a record in the fork's subspace, which every start
|
||||
// writes (here whichever one the test server wrote first)
|
||||
let mut startup_key = None;
|
||||
db.iterate(
|
||||
IterateParams::new(
|
||||
AnyKey {
|
||||
subspace: SUBSPACE_INBUXA,
|
||||
key: vec![0u8],
|
||||
},
|
||||
AnyKey {
|
||||
subspace: SUBSPACE_INBUXA,
|
||||
key: vec![u8::MAX; 32],
|
||||
},
|
||||
)
|
||||
.no_values(),
|
||||
|key, _| {
|
||||
startup_key = Some(key.to_vec());
|
||||
Ok(false)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let startup_key =
|
||||
startup_key.expect("the test server's start wrote nothing to the fork's subspace");
|
||||
|
||||
// Destroy store
|
||||
println!("Destroying store...");
|
||||
store_destroy(&db).await;
|
||||
@@ -293,6 +318,16 @@ pub async fn test(test: &TestServer) {
|
||||
.write("localhost")
|
||||
.finalize(),
|
||||
);
|
||||
// inbuxa: K-C11: and over what any start writes first, the import's own
|
||||
// too: a record in the fork's subspace, with a value the import has to
|
||||
// set over. Refused as data in the store until rc.34.
|
||||
batch.set(
|
||||
ValueClass::Any(AnyClass {
|
||||
subspace: SUBSPACE_INBUXA,
|
||||
key: startup_key,
|
||||
}),
|
||||
b"written at start-up".to_vec(),
|
||||
);
|
||||
db.write(batch.build_all()).await.unwrap();
|
||||
for (key, _) in &named_blobs {
|
||||
test.server.blob_store().delete_blob(key).await.unwrap();
|
||||
|
||||
Reference in new issue
Block a user