Files
inbuxa-migrate/src/migrate/mod.rs
T
jcoffey-dev a50733c924
ci / test (pull_request) Skipped
github/ci (branch) GitHub Actions
ci / github (pull_request) Successful in 4m50s
ci / announce (pull_request) Skipped
Add migrate --plan: many accounts moved by one plan file
A plan names setup steps (import-directory or export-directory, run once),
the steps for each account, and the accounts, listed or taken from the
users in a directory.json. A step is one inbuxa-migrate command line as
the docs write it, with {email}, {local}, {domain} and {archive} filled in
per account, so every source the tool reads works with no new code.

Each step runs as a child of the same binary in the plan's directory,
with stdin closed and only the secrets it names: secrets are given by
environment variable, all others are cleared, and a plan that writes a
password or token into its args is refused. Output goes to an append-only
log per account and step; --parallel accounts run at once.

The work directory (mode 700) holds an archive per account, state.json
and report.csv. A second run skips finished accounts and, for the rest,
runs from the first step that wasn't done; --again runs every step once
more for the final pass. A failed step stops its account, a partial one
doesn't, and a failed setup step stops the run.
2026-10-30 09:20:28 -07:00

537 lines
16 KiB
Rust

/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC <[email protected]>
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*/
//! `migrate`: many accounts moved by one plan. Each step is a run of this
//! same binary, so it behaves exactly as the command the docs describe, its
//! output goes to that account's log, and one account failing leaves the
//! rest running. The work directory holds an archive and logs per account,
//! the state a second run resumes from, and the report.
pub mod plan;
use std::collections::BTreeMap;
use std::fs::{self, File, OpenOptions};
use std::io::{Read, Seek, SeekFrom, Write};
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::sync::Mutex;
use std::sync::atomic::{AtomicUsize, Ordering};
use serde::{Deserialize, Serialize};
use crate::error::Error;
use crate::logging::Logger;
use plan::{Plan, SECRET_VARS, Step};
pub struct MigrateConfig {
pub plan: PathBuf,
pub work: PathBuf,
/// Accounts migrated at once.
pub parallel: usize,
/// Run every account step again, done or not.
pub again: bool,
/// Only these accounts; empty for all of them.
pub only: Vec<String>,
pub dry_run: bool,
/// The binary each step runs: this one.
pub program: PathBuf,
pub logger: Logger,
}
pub struct Outcome {
pub done: usize,
pub unfinished: usize,
pub report: PathBuf,
}
const STATE_FORMAT: &str = "inbuxa-migrate-state";
#[derive(Serialize, Deserialize, Default)]
#[serde(rename_all = "camelCase")]
struct State {
format: String,
format_version: u32,
setup: BTreeMap<String, Ended>,
accounts: BTreeMap<String, BTreeMap<String, Ended>>,
}
/// How a step ended. Only `Done` is skipped by the next run.
#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Debug)]
#[serde(rename_all = "camelCase")]
enum Ended {
Done,
Partial,
Failed,
}
/// What one step did this run, for the report.
struct StepReport {
step: String,
result: &'static str,
counts: Counts,
detail: String,
}
#[derive(Default, Clone, Copy)]
struct Counts {
fetched: u64,
created: u64,
updated: u64,
deleted: u64,
skipped: u64,
failed: u64,
}
struct Ran {
result: Ended,
counts: Counts,
detail: String,
}
pub fn run(config: MigrateConfig) -> Result<Outcome, Error> {
let plan = plan::load(&config.plan)?;
let accounts = chosen(&plan, &config.only)?;
if config.parallel == 0 {
return Err(Error::Usage("--parallel must be at least 1".to_owned()));
}
let work = std::path::absolute(&config.work)?;
private_dir(&work)?;
let state_path = work.join("state.json");
let state = Mutex::new(load_state(&state_path)?);
let save = |state: &State| -> Result<(), Error> {
if config.dry_run {
return Ok(());
}
let tmp = work.join("state.json.tmp");
fs::write(&tmp, serde_json::to_vec_pretty(state).map_err(io_err)?)?;
fs::rename(&tmp, &state_path)?;
Ok(())
};
// Setup runs once, before any account, and a failure stops the run:
// the accounts need what it makes.
let mut setup_reports = Vec::new();
if !plan.setup.is_empty() {
private_dir(&work.join("setup"))?;
}
for step in &plan.setup {
if lock(&state).setup.get(&step.name) == Some(&Ended::Done) {
setup_reports.push(earlier(&step.name));
continue;
}
let log = work.join("setup").join(format!("{}.log", step.name));
let ran = run_step(&config, &plan, step, None, &log);
setup_reports.push(report_of(&step.name, &ran));
if !config.dry_run {
lock(&state).setup.insert(step.name.clone(), ran.result);
}
save(&lock(&state))?;
if ran.result == Ended::Failed {
write_report(&work, &config, &setup_reports, &[])?;
return Err(Error::Partial(format!(
"setup step {} failed: {}; see {}",
step.name,
ran.detail,
log.display()
)));
}
}
let next = AtomicUsize::new(0);
let finished = AtomicUsize::new(0);
let reports: Mutex<Vec<(String, Vec<StepReport>)>> = Mutex::new(Vec::new());
let failure: Mutex<Option<Error>> = Mutex::new(None);
let total = accounts.len();
std::thread::scope(|scope| {
for _ in 0..config.parallel.min(total) {
scope.spawn(|| {
loop {
let i = next.fetch_add(1, Ordering::SeqCst);
let Some(email) = accounts.get(i) else { break };
match migrate_account(&config, &plan, &work, email, &state, &save) {
Ok(steps) => {
let n = finished.fetch_add(1, Ordering::SeqCst) + 1;
if config.logger.enabled(crate::logging::LEVEL_DEFAULT) {
eprintln!("[{n}/{total}] {email}: {}", line_of(&steps));
}
lock(&reports).push((email.clone(), steps));
}
Err(e) => {
// The state can't be saved: stop handing out
// accounts, so nothing runs unrecorded.
next.store(total, Ordering::SeqCst);
lock(&failure).get_or_insert(e);
}
}
}
});
}
});
if let Some(e) = lock(&failure).take() {
return Err(e);
}
let mut reports = reports.into_inner().unwrap_or_else(|e| e.into_inner());
reports.sort_by_key(|(email, _)| accounts.iter().position(|a| a == email));
let unfinished = reports
.iter()
.filter(|(_, steps)| {
steps
.iter()
.any(|s| s.result != "done" && s.result != "done earlier")
})
.count();
let report = write_report(&work, &config, &setup_reports, &reports)?;
Ok(Outcome {
done: reports.len() - unfinished,
unfinished,
report,
})
}
fn lock<T>(m: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
m.lock().unwrap_or_else(|e| e.into_inner())
}
fn io_err(e: serde_json::Error) -> Error {
Error::Io(std::io::Error::other(e))
}
fn chosen(plan: &Plan, only: &[String]) -> Result<Vec<String>, Error> {
if only.is_empty() {
return Ok(plan.accounts.clone());
}
let mut out = Vec::new();
for email in only {
let email = email.trim().to_ascii_lowercase();
if !plan.accounts.contains(&email) {
return Err(Error::Usage(format!(
"--account {email} is not in the plan"
)));
}
if !out.contains(&email) {
out.push(email);
}
}
Ok(out)
}
/// The work directory holds whole mailboxes: readable by its owner only.
fn private_dir(path: &Path) -> Result<(), Error> {
if path.is_dir() {
return Ok(());
}
let mut builder = fs::DirBuilder::new();
builder.recursive(true);
#[cfg(unix)]
{
use std::os::unix::fs::DirBuilderExt;
builder.mode(0o700);
}
builder.create(path)?;
Ok(())
}
fn load_state(path: &Path) -> Result<State, Error> {
let text = match fs::read_to_string(path) {
Ok(text) => text,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
return Ok(State {
format: STATE_FORMAT.to_owned(),
format_version: 1,
..State::default()
});
}
Err(e) => return Err(e.into()),
};
let state: State = serde_json::from_str(&text)
.map_err(|e| Error::Usage(format!("{}: {e}", path.display())))?;
if state.format != STATE_FORMAT || state.format_version != 1 {
return Err(Error::Usage(format!(
"{} is not a migrate state file this release reads",
path.display()
)));
}
Ok(state)
}
/// One account's steps. Done steps at the front are skipped; from the first
/// that isn't done, every step runs, so an import that picked up more is
/// always followed by the export that delivers it. A step that fails stops
/// the account; a partial one doesn't.
fn migrate_account(
config: &MigrateConfig,
plan: &Plan,
work: &Path,
email: &str,
state: &Mutex<State>,
save: &(dyn Fn(&State) -> Result<(), Error> + Sync),
) -> Result<Vec<StepReport>, Error> {
let dir = work.join(email);
private_dir(&dir)?;
let archive = dir.join("archive.sqlite");
let mut reports = Vec::new();
let mut running = config.again;
let mut stopped = false;
for step in &plan.steps {
if stopped {
reports.push(StepReport {
step: step.name.clone(),
result: "not run",
counts: Counts::default(),
detail: "an earlier step failed".to_owned(),
});
continue;
}
let was = lock(state)
.accounts
.get(email)
.and_then(|steps| steps.get(&step.name))
.copied();
if !running && was == Some(Ended::Done) {
reports.push(earlier(&step.name));
continue;
}
running = true;
let log = dir.join(format!("{}.log", step.name));
let ran = run_step(config, plan, step, Some((email, &archive)), &log);
reports.push(report_of(&step.name, &ran));
if !config.dry_run {
let mut s = lock(state);
s.accounts
.entry(email.to_owned())
.or_default()
.insert(step.name.clone(), ran.result);
save(&s)?;
}
stopped = ran.result == Ended::Failed;
}
Ok(reports)
}
fn earlier(step: &str) -> StepReport {
StepReport {
step: step.to_owned(),
result: "done earlier",
counts: Counts::default(),
detail: String::new(),
}
}
fn report_of(step: &str, ran: &Ran) -> StepReport {
StepReport {
step: step.to_owned(),
result: match ran.result {
Ended::Done => "done",
Ended::Partial => "partial",
Ended::Failed => "failed",
},
counts: ran.counts,
detail: ran.detail.clone(),
}
}
fn line_of(steps: &[StepReport]) -> String {
match steps
.iter()
.find(|s| s.result == "failed" || s.result == "partial")
{
Some(s) if s.detail.is_empty() => format!("{} at {}", s.result, s.step),
Some(s) => format!("{} at {}: {}", s.result, s.step, s.detail),
None => "done".to_owned(),
}
}
/// Runs one step as a child: no terminal, so a missing secret fails
/// rather than waits; only the secrets the step names; stderr to the log,
/// appended so a resumed run keeps the history.
fn run_step(
config: &MigrateConfig,
plan: &Plan,
step: &Step,
account: Option<(&str, &Path)>,
log: &Path,
) -> Ran {
let failed = |detail: String| Ran {
result: Ended::Failed,
counts: Counts::default(),
detail,
};
let mut args = step.args_for(account);
if config.dry_run {
args.push("--dry-run".to_owned());
}
let (mut file, start) = match open_log(log) {
Ok(opened) => opened,
Err(e) => return failed(format!("can't write {}: {e}", log.display())),
};
// The args hold no secrets; the plan refuses them.
let _ = writeln!(file, "== inbuxa-migrate {}", args.join(" "));
let stderr = match file.try_clone() {
Ok(f) => f,
Err(e) => return failed(format!("can't write {}: {e}", log.display())),
};
let mut command = Command::new(&config.program);
command
.args(&args)
.current_dir(&plan.base)
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::from(stderr));
for var in SECRET_VARS {
command.env_remove(var);
}
for (var, value) in &step.env {
command.env(var, value);
}
let output = match command.output() {
Ok(output) => output,
Err(e) => return failed(format!("can't run {}: {e}", config.program.display())),
};
let stdout = String::from_utf8_lossy(&output.stdout);
let _ = file.write_all(stdout.as_bytes());
let counts = counts_of(&stdout);
let result = match output.status.code() {
Some(0) => Ended::Done,
Some(5) => Ended::Partial,
_ => Ended::Failed,
};
let detail = match result {
Ended::Done => String::new(),
_ => last_error(log, start).unwrap_or_else(|| match output.status.code() {
Some(code) => format!("exit code {code}"),
None => "stopped by a signal".to_owned(),
}),
};
Ran {
result,
counts,
detail,
}
}
fn open_log(path: &Path) -> std::io::Result<(File, u64)> {
let mut options = OpenOptions::new();
options.create(true).append(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.mode(0o600);
}
let file = options.open(path)?;
let start = file.metadata()?.len();
Ok((file, start))
}
/// The last `error:` line this run wrote to the log.
fn last_error(log: &Path, start: u64) -> Option<String> {
let mut file = File::open(log).ok()?;
file.seek(SeekFrom::Start(start)).ok()?;
let mut text = String::new();
file.read_to_string(&mut text).ok()?;
text.lines()
.rev()
.find_map(|l| l.strip_prefix("error: "))
.map(str::to_owned)
}
/// The per-type lines a step prints, summed:
/// `email: created=1 fetched=2 updated=0 deleted=0 skipped=0 failed=0`.
fn counts_of(stdout: &str) -> Counts {
let mut c = Counts::default();
for line in stdout.lines() {
let Some((_, rest)) = line.split_once(": ") else {
continue;
};
for field in rest.split_whitespace() {
let Some((key, n)) = field.split_once('=') else {
continue;
};
let Ok(n) = n.parse::<u64>() else { continue };
match key {
"fetched" => c.fetched += n,
"created" => c.created += n,
"updated" => c.updated += n,
"deleted" => c.deleted += n,
"skipped" => c.skipped += n,
"failed" => c.failed += n,
_ => {}
}
}
}
c
}
/// `report.csv` in the work directory (`dry-run-report.csv` for a dry
/// run): one row per step per account, setup first.
fn write_report(
work: &Path,
config: &MigrateConfig,
setup: &[StepReport],
accounts: &[(String, Vec<StepReport>)],
) -> Result<PathBuf, Error> {
let path = work.join(if config.dry_run {
"dry-run-report.csv"
} else {
"report.csv"
});
let mut out =
String::from("account,step,result,fetched,created,updated,deleted,skipped,failed,detail\n");
let rows = setup.iter().map(|s| ("(setup)", s)).chain(
accounts
.iter()
.flat_map(|(email, steps)| steps.iter().map(move |s| (email.as_str(), s))),
);
for (account, s) in rows {
let c = s.counts;
out.push_str(&format!(
"{account},{},{},{},{},{},{},{},{},{}\n",
s.step,
s.result,
c.fetched,
c.created,
c.updated,
c.deleted,
c.skipped,
c.failed,
csv_field(&s.detail)
));
}
fs::write(&path, out)?;
Ok(path)
}
fn csv_field(s: &str) -> String {
if s.contains([',', '"', '\n', '\r']) {
format!("\"{}\"", s.replace('"', "\"\""))
} else {
s.to_owned()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn counts_are_summed_over_the_types() {
let c = counts_of(
"email: created=3 fetched=4 updated=1 deleted=0 skipped=2 failed=1\n\
mailbox: created=2 fetched=2 updated=0 deleted=0 skipped=0 failed=0\n",
);
assert_eq!((c.fetched, c.created, c.updated), (6, 5, 1));
assert_eq!((c.skipped, c.failed), (2, 1));
}
#[test]
fn details_are_quoted_when_they_need_it() {
assert_eq!(csv_field("plain"), "plain");
assert_eq!(csv_field("a, \"b\""), "\"a, \"\"b\"\"\"");
}
}