From a50733c924bdd8acbc2812ad1a8e99c7488007f8 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Fri, 30 Oct 2026 09:20:28 -0700 Subject: [PATCH] Add migrate --plan: many accounts moved by one plan file A plan names setup steps (import-directory or export-directory, run once), the steps for each account, and the accounts, listed or taken from the users in a directory.json. A step is one inbuxa-migrate command line as the docs write it, with {email}, {local}, {domain} and {archive} filled in per account, so every source the tool reads works with no new code. Each step runs as a child of the same binary in the plan's directory, with stdin closed and only the secrets it names: secrets are given by environment variable, all others are cleared, and a plan that writes a password or token into its args is refused. Output goes to an append-only log per account and step; --parallel accounts run at once. The work directory (mode 700) holds an archive per account, state.json and report.csv. A second run skips finished accounts and, for the rest, runs from the first step that wasn't done; --again runs every step once more for the final pass. A failed step stops its account, a partial one doesn't, and a failed setup step stops the run. --- CHANGELOG.md | 8 + README.md | 7 +- docs/usage.md | 81 ++++++- src/cli.rs | 64 +++++ src/lib.rs | 1 + src/main.rs | 24 +- src/migrate/mod.rs | 536 ++++++++++++++++++++++++++++++++++++++++++ src/migrate/plan.rs | 430 +++++++++++++++++++++++++++++++++ tests/mock_migrate.rs | 180 ++++++++++++++ 9 files changed, 1327 insertions(+), 4 deletions(-) create mode 100644 src/migrate/mod.rs create mode 100644 src/migrate/plan.rs create mode 100644 tests/mock_migrate.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 4028618..0e6b865 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,14 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a is left as it is, so running it again creates nothing. Each user created gets a generated password, written to a new file readable by its owner only. +- `migrate --plan` moves many accounts with one command. The plan names + the steps, which are the commands in the docs with `{email}` and + `{archive}` filled in per account, and the accounts, listed or taken + from `directory.json`. Several accounts run at once, each with its own + archive and logs. A second run resumes where each stopped, `--again` + runs the final pass, and `report.csv` says what moved and why anything + didn't. Secrets are named by environment variable, never written in the + plan. ### Fixed - `inspect` opens the archive read-only. It used to create an empty archive diff --git a/README.md b/README.md index 245904d..7598ed5 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ contacts, identities, Sieve scripts and files. inbuxa-migrate export-carddav … alice.sqlite its contacts into any CardDAV server inbuxa-migrate export-directory … out a server's accounts, no secrets inbuxa-migrate import-directory … out those accounts, set up on inbuxa + inbuxa-migrate migrate --plan plan.json work many accounts, by a plan Between the two sides sits an archive: one SQLite file holding one account. `import` fills it from the old server; `export` writes it into the new one. @@ -49,8 +50,10 @@ servers and mail, calendar and contacts apps open, with no network and no store format in the way. For a whole server, `export-directory` writes the list of domains, accounts, groups, lists and aliases to set up again elsewhere, without passwords, and `import-directory` sets them up on an -inbuxa server, with a new password for each user. The full command -reference is in [docs/usage.md](docs/usage.md). +inbuxa server, with a new password for each user. `migrate` moves many +accounts with one plan file: the same steps for each account, several at +once, resuming where it stopped, with one report at the end. The full +command reference is in [docs/usage.md](docs/usage.md). ## Sources diff --git a/docs/usage.md b/docs/usage.md index 22a5f37..1b403ad 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -13,7 +13,7 @@ and how to install it. ## Commands and global flags ``` -inbuxa-migrate [args...] +inbuxa-migrate [args...] ``` Every command takes a few global flags -- verbosity, worker pool size, retry @@ -551,6 +551,85 @@ point at it, so a user finds its domain, its groups and its roles. server and counts what it would create, sends nothing and writes no file; `--passwords` isn't needed then. +## Migrate many accounts + +``` +inbuxa-migrate migrate --plan [--parallel ] [--again] + [--account ]... [--dry-run] [-q] +``` + +Moves a list of accounts with one command. The plan file names the steps +and the accounts. Each step is one of the commands on this page, written +the same way, and `migrate` runs it for every account, several accounts at +once. `` is the work directory. It holds an archive and a log per +account, the state a later run resumes from, and the report. + +```json +{ + "format": "inbuxa-migrate-plan", + "formatVersion": 1, + "setup": [ + {"name": "directory", + "args": ["import-directory", "--url", "https://mail.example.org", + "--auth-bearer", "--passwords", "passwords.csv", "directory"], + "secrets": {"token": "INBUXA_ADMIN_TOKEN"}} + ], + "accounts": {"directory": "directory/directory.json"}, + "steps": [ + {"name": "mail", + "args": ["import", "imap", "--url", "imaps://old.example.org", + "--auth-basic", "{email}*master", "{archive}"], + "secrets": {"password": "OLD_MASTER_PASSWORD"}}, + {"name": "inbuxa", + "args": ["export", "--url", "https://mail.example.org", "--auth-bearer", + "--account-name", "{email}", "{archive}"], + "secrets": {"token": "INBUXA_ADMIN_TOKEN"}} + ] +} +``` + +- **Steps.** `setup` steps run once, before any account: + `import-directory` or `export-directory`. The `steps` run for each + account, in order, and each has to name `{archive}`. `{email}`, + `{local}` and `{domain}` are the account's address and its two halves. + Steps run in the plan's directory, so a relative path means the same as + it does in the plan. +- **Accounts.** A list of addresses, or `{"directory": "…/directory.json"}` + for the users that `export-directory` wrote. +- **Secrets stay out of the plan.** A step names the environment variable + that holds each secret it needs: `password`, `token`, `clientSecret` + (EWS) or `graphToken`. A step sees only the secrets it names. A plan that + writes a password or token into `args` is refused. A step that needs a + secret it doesn't name fails at once instead of waiting for a prompt. +- **Signing in for each account.** Each step signs in the way its command + does, so the plan decides how. To avoid collecting every user's password, + use a login that can act for each account. The example uses Dovecot's + master user, `{email}*master`. Use the source's own form where it has + one: Exchange impersonation through `--mailbox {email}`, or an + administrator on a JMAP source with `--account-name {email}`. +- **One account at a time, several at once.** `--parallel` (default 4) + sets how many accounts run at the same time. Each step runs as its own + process. Its output goes to `//.log`, and one account + failing doesn't stop the others. A step that fails stops its account. + A step that leaves some objects behind (exit code 5) doesn't. A failed + setup step stops the run. +- **Resuming.** A second run with the same plan and work directory skips + every account that finished. For the rest, it skips the steps that were + done and runs everything from the first one that wasn't. `--again` runs + every account step once more, done or not. Use it for the final pass + once the old server stops taking mail: the steps converge, so it only + moves what changed. `--account` limits a run to the accounts named. +- **The report.** `/report.csv` has one row per step per account: + how it ended (`done`, `partial`, `failed`, `not run`, or + `done earlier`), the counts summed over the step's types, and, for one + that didn't finish, its last error. Exit code 5 means some account + didn't finish. + +The work directory is made readable by its owner only, and so are the +logs. With `--dry-run`, each step runs with `--dry-run`, nothing is +recorded, and the report goes to `dry-run-report.csv`. An export step +reads the archive as it is, so on a first dry run it has nothing to show. + ## Inspect ``` diff --git a/src/cli.rs b/src/cli.rs index 75994cd..c3f7bf4 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -21,6 +21,7 @@ use crate::inspect::InspectConfig; use crate::jmap::account::AccountSelector; use crate::jmap::http::Auth; use crate::logging::Logger; +use crate::migrate::MigrateConfig; use crate::secret; use crate::sync::export_dav::ExportDavConfig; use crate::sync::export_imap::ExportImapConfig; @@ -90,6 +91,10 @@ enum Command { name = "import-directory" )] ImportDirectory(Box), + #[command( + about = "Move many accounts by a plan file: the same steps for each, several at once, resumable" + )] + Migrate(Box), #[command(about = "Show the contents of a local archive (read-only)")] Inspect(Box), } @@ -684,6 +689,50 @@ struct ImportDirectoryArgs { dir: PathBuf, } +#[derive(Args)] +struct MigrateArgs { + #[arg( + long, + value_name = "FILE", + help = "Plan file (JSON): the setup steps, the steps for each account, and the accounts" + )] + plan: PathBuf, + + #[arg( + long, + value_name = "N", + default_value_t = 4, + help = "Accounts migrated at once" + )] + parallel: usize, + + #[arg( + long, + help = "Run every account step again, done or not: the final pass once the old server stops taking mail" + )] + again: bool, + + #[arg( + long = "account", + value_name = "EMAIL", + action = ArgAction::Append, + help = "Only this account (repeatable; default: every account in the plan)" + )] + accounts: Vec, + + #[arg(long, help = "Run each step with --dry-run; record nothing")] + dry_run: bool, + + #[arg(short = 'q', long, help = "Quiet: no line per account")] + quiet: bool, + + #[arg( + value_name = "DIR", + help = "Work directory: an archive and logs per account, the state and the report" + )] + work: PathBuf, +} + #[derive(Args)] struct InspectArgs { #[arg(value_name = "ARCHIVE", help = "Local SQLite archive")] @@ -724,6 +773,7 @@ pub enum Action { ExportFiles(ExportFilesConfig), ExportDirectory(ExportDirectoryConfig), ImportDirectory(ImportDirectoryConfig), + Migrate(MigrateConfig), ExportImap(CommonConfig, ExportImapConfig), ExportManageSieve(CommonConfig, ExportManageSieveConfig), ExportDav(CommonConfig, ExportDavConfig), @@ -855,6 +905,7 @@ impl Cli { Command::ExportFiles(args) => resolve_export_files(*args), Command::ExportDirectory(args) => resolve_export_directory(*args), Command::ImportDirectory(args) => resolve_import_directory(*args), + Command::Migrate(args) => resolve_migrate(*args), Command::ExportImap(args) => resolve_export_imap(*args), Command::ExportManageSieve(args) => resolve_export_managesieve(*args), Command::ExportCaldav(args) => resolve_export_dav(*args, DavKindArg::Caldav), @@ -1339,6 +1390,19 @@ fn resolve_import_directory(args: ImportDirectoryArgs) -> Result })) } +fn resolve_migrate(args: MigrateArgs) -> Result { + Ok(Action::Migrate(MigrateConfig { + plan: args.plan, + work: args.work, + parallel: args.parallel, + again: args.again, + only: args.accounts, + dry_run: args.dry_run, + program: std::env::current_exe()?, + logger: Logger::from_flags(args.quiet, 0), + })) +} + fn resolve_inspect(args: InspectArgs) -> Result { let target = match args.target.as_deref() { Some(token) => Some(ObjectType::parse(token)?), diff --git a/src/lib.rs b/src/lib.rs index 62b2732..f11c6b5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,6 +20,7 @@ pub mod inspect; pub mod jmap; pub mod logging; pub mod managesieve; +pub mod migrate; pub mod net; pub mod secret; pub mod sync; diff --git a/src/main.rs b/src/main.rs index e654fb0..8210d56 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,7 +10,7 @@ use clap::Parser; use inbuxa_migrate::cli::{Action, Cli}; use inbuxa_migrate::error::Error; use inbuxa_migrate::sync::{self, RunOutcome, Summary}; -use inbuxa_migrate::{export_directory, export_files, import_directory, inspect}; +use inbuxa_migrate::{export_directory, export_files, import_directory, inspect, migrate}; fn main() { let code = run(); @@ -125,6 +125,28 @@ fn run() -> i32 { logger, ) } + Action::Migrate(config) => { + return match migrate::run(config) { + Ok(outcome) => { + println!( + "accounts: {} done, {} not finished; report: {}", + outcome.done, + outcome.unfinished, + outcome.report.display() + ); + if outcome.unfinished > 0 { + eprintln!( + "error: some accounts did not finish; the report says why. Fix that \ + and run the same command again: it resumes where each stopped" + ); + 5 + } else { + 0 + } + } + Err(err) => fail(&err), + }; + } Action::Inspect(config) => { return match inspect::run(config) { Ok(()) => 0, diff --git a/src/migrate/mod.rs b/src/migrate/mod.rs new file mode 100644 index 0000000..04bfdf7 --- /dev/null +++ b/src/migrate/mod.rs @@ -0,0 +1,536 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! `migrate`: many accounts moved by one plan. Each step is a run of this +//! same binary, so it behaves exactly as the command the docs describe, its +//! output goes to that account's log, and one account failing leaves the +//! rest running. The work directory holds an archive and logs per account, +//! the state a second run resumes from, and the report. + +pub mod plan; + +use std::collections::BTreeMap; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::Mutex; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use serde::{Deserialize, Serialize}; + +use crate::error::Error; +use crate::logging::Logger; + +use plan::{Plan, SECRET_VARS, Step}; + +pub struct MigrateConfig { + pub plan: PathBuf, + pub work: PathBuf, + /// Accounts migrated at once. + pub parallel: usize, + /// Run every account step again, done or not. + pub again: bool, + /// Only these accounts; empty for all of them. + pub only: Vec, + pub dry_run: bool, + /// The binary each step runs: this one. + pub program: PathBuf, + pub logger: Logger, +} + +pub struct Outcome { + pub done: usize, + pub unfinished: usize, + pub report: PathBuf, +} + +const STATE_FORMAT: &str = "inbuxa-migrate-state"; + +#[derive(Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +struct State { + format: String, + format_version: u32, + setup: BTreeMap, + accounts: BTreeMap>, +} + +/// How a step ended. Only `Done` is skipped by the next run. +#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Debug)] +#[serde(rename_all = "camelCase")] +enum Ended { + Done, + Partial, + Failed, +} + +/// What one step did this run, for the report. +struct StepReport { + step: String, + result: &'static str, + counts: Counts, + detail: String, +} + +#[derive(Default, Clone, Copy)] +struct Counts { + fetched: u64, + created: u64, + updated: u64, + deleted: u64, + skipped: u64, + failed: u64, +} + +struct Ran { + result: Ended, + counts: Counts, + detail: String, +} + +pub fn run(config: MigrateConfig) -> Result { + let plan = plan::load(&config.plan)?; + let accounts = chosen(&plan, &config.only)?; + if config.parallel == 0 { + return Err(Error::Usage("--parallel must be at least 1".to_owned())); + } + let work = std::path::absolute(&config.work)?; + private_dir(&work)?; + + let state_path = work.join("state.json"); + let state = Mutex::new(load_state(&state_path)?); + let save = |state: &State| -> Result<(), Error> { + if config.dry_run { + return Ok(()); + } + let tmp = work.join("state.json.tmp"); + fs::write(&tmp, serde_json::to_vec_pretty(state).map_err(io_err)?)?; + fs::rename(&tmp, &state_path)?; + Ok(()) + }; + + // Setup runs once, before any account, and a failure stops the run: + // the accounts need what it makes. + let mut setup_reports = Vec::new(); + if !plan.setup.is_empty() { + private_dir(&work.join("setup"))?; + } + for step in &plan.setup { + if lock(&state).setup.get(&step.name) == Some(&Ended::Done) { + setup_reports.push(earlier(&step.name)); + continue; + } + let log = work.join("setup").join(format!("{}.log", step.name)); + let ran = run_step(&config, &plan, step, None, &log); + setup_reports.push(report_of(&step.name, &ran)); + if !config.dry_run { + lock(&state).setup.insert(step.name.clone(), ran.result); + } + save(&lock(&state))?; + if ran.result == Ended::Failed { + write_report(&work, &config, &setup_reports, &[])?; + return Err(Error::Partial(format!( + "setup step {} failed: {}; see {}", + step.name, + ran.detail, + log.display() + ))); + } + } + + let next = AtomicUsize::new(0); + let finished = AtomicUsize::new(0); + let reports: Mutex)>> = Mutex::new(Vec::new()); + let failure: Mutex> = Mutex::new(None); + let total = accounts.len(); + std::thread::scope(|scope| { + for _ in 0..config.parallel.min(total) { + scope.spawn(|| { + loop { + let i = next.fetch_add(1, Ordering::SeqCst); + let Some(email) = accounts.get(i) else { break }; + match migrate_account(&config, &plan, &work, email, &state, &save) { + Ok(steps) => { + let n = finished.fetch_add(1, Ordering::SeqCst) + 1; + if config.logger.enabled(crate::logging::LEVEL_DEFAULT) { + eprintln!("[{n}/{total}] {email}: {}", line_of(&steps)); + } + lock(&reports).push((email.clone(), steps)); + } + Err(e) => { + // The state can't be saved: stop handing out + // accounts, so nothing runs unrecorded. + next.store(total, Ordering::SeqCst); + lock(&failure).get_or_insert(e); + } + } + } + }); + } + }); + if let Some(e) = lock(&failure).take() { + return Err(e); + } + + let mut reports = reports.into_inner().unwrap_or_else(|e| e.into_inner()); + reports.sort_by_key(|(email, _)| accounts.iter().position(|a| a == email)); + let unfinished = reports + .iter() + .filter(|(_, steps)| { + steps + .iter() + .any(|s| s.result != "done" && s.result != "done earlier") + }) + .count(); + let report = write_report(&work, &config, &setup_reports, &reports)?; + Ok(Outcome { + done: reports.len() - unfinished, + unfinished, + report, + }) +} + +fn lock(m: &Mutex) -> std::sync::MutexGuard<'_, T> { + m.lock().unwrap_or_else(|e| e.into_inner()) +} + +fn io_err(e: serde_json::Error) -> Error { + Error::Io(std::io::Error::other(e)) +} + +fn chosen(plan: &Plan, only: &[String]) -> Result, Error> { + if only.is_empty() { + return Ok(plan.accounts.clone()); + } + let mut out = Vec::new(); + for email in only { + let email = email.trim().to_ascii_lowercase(); + if !plan.accounts.contains(&email) { + return Err(Error::Usage(format!( + "--account {email} is not in the plan" + ))); + } + if !out.contains(&email) { + out.push(email); + } + } + Ok(out) +} + +/// The work directory holds whole mailboxes: readable by its owner only. +fn private_dir(path: &Path) -> Result<(), Error> { + if path.is_dir() { + return Ok(()); + } + let mut builder = fs::DirBuilder::new(); + builder.recursive(true); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder.create(path)?; + Ok(()) +} + +fn load_state(path: &Path) -> Result { + let text = match fs::read_to_string(path) { + Ok(text) => text, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + return Ok(State { + format: STATE_FORMAT.to_owned(), + format_version: 1, + ..State::default() + }); + } + Err(e) => return Err(e.into()), + }; + let state: State = serde_json::from_str(&text) + .map_err(|e| Error::Usage(format!("{}: {e}", path.display())))?; + if state.format != STATE_FORMAT || state.format_version != 1 { + return Err(Error::Usage(format!( + "{} is not a migrate state file this release reads", + path.display() + ))); + } + Ok(state) +} + +/// One account's steps. Done steps at the front are skipped; from the first +/// that isn't done, every step runs, so an import that picked up more is +/// always followed by the export that delivers it. A step that fails stops +/// the account; a partial one doesn't. +fn migrate_account( + config: &MigrateConfig, + plan: &Plan, + work: &Path, + email: &str, + state: &Mutex, + save: &(dyn Fn(&State) -> Result<(), Error> + Sync), +) -> Result, Error> { + let dir = work.join(email); + private_dir(&dir)?; + let archive = dir.join("archive.sqlite"); + let mut reports = Vec::new(); + let mut running = config.again; + let mut stopped = false; + for step in &plan.steps { + if stopped { + reports.push(StepReport { + step: step.name.clone(), + result: "not run", + counts: Counts::default(), + detail: "an earlier step failed".to_owned(), + }); + continue; + } + let was = lock(state) + .accounts + .get(email) + .and_then(|steps| steps.get(&step.name)) + .copied(); + if !running && was == Some(Ended::Done) { + reports.push(earlier(&step.name)); + continue; + } + running = true; + let log = dir.join(format!("{}.log", step.name)); + let ran = run_step(config, plan, step, Some((email, &archive)), &log); + reports.push(report_of(&step.name, &ran)); + if !config.dry_run { + let mut s = lock(state); + s.accounts + .entry(email.to_owned()) + .or_default() + .insert(step.name.clone(), ran.result); + save(&s)?; + } + stopped = ran.result == Ended::Failed; + } + Ok(reports) +} + +fn earlier(step: &str) -> StepReport { + StepReport { + step: step.to_owned(), + result: "done earlier", + counts: Counts::default(), + detail: String::new(), + } +} + +fn report_of(step: &str, ran: &Ran) -> StepReport { + StepReport { + step: step.to_owned(), + result: match ran.result { + Ended::Done => "done", + Ended::Partial => "partial", + Ended::Failed => "failed", + }, + counts: ran.counts, + detail: ran.detail.clone(), + } +} + +fn line_of(steps: &[StepReport]) -> String { + match steps + .iter() + .find(|s| s.result == "failed" || s.result == "partial") + { + Some(s) if s.detail.is_empty() => format!("{} at {}", s.result, s.step), + Some(s) => format!("{} at {}: {}", s.result, s.step, s.detail), + None => "done".to_owned(), + } +} + +/// Runs one step as a child: no terminal, so a missing secret fails +/// rather than waits; only the secrets the step names; stderr to the log, +/// appended so a resumed run keeps the history. +fn run_step( + config: &MigrateConfig, + plan: &Plan, + step: &Step, + account: Option<(&str, &Path)>, + log: &Path, +) -> Ran { + let failed = |detail: String| Ran { + result: Ended::Failed, + counts: Counts::default(), + detail, + }; + let mut args = step.args_for(account); + if config.dry_run { + args.push("--dry-run".to_owned()); + } + let (mut file, start) = match open_log(log) { + Ok(opened) => opened, + Err(e) => return failed(format!("can't write {}: {e}", log.display())), + }; + // The args hold no secrets; the plan refuses them. + let _ = writeln!(file, "== inbuxa-migrate {}", args.join(" ")); + let stderr = match file.try_clone() { + Ok(f) => f, + Err(e) => return failed(format!("can't write {}: {e}", log.display())), + }; + let mut command = Command::new(&config.program); + command + .args(&args) + .current_dir(&plan.base) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::from(stderr)); + for var in SECRET_VARS { + command.env_remove(var); + } + for (var, value) in &step.env { + command.env(var, value); + } + let output = match command.output() { + Ok(output) => output, + Err(e) => return failed(format!("can't run {}: {e}", config.program.display())), + }; + let stdout = String::from_utf8_lossy(&output.stdout); + let _ = file.write_all(stdout.as_bytes()); + let counts = counts_of(&stdout); + let result = match output.status.code() { + Some(0) => Ended::Done, + Some(5) => Ended::Partial, + _ => Ended::Failed, + }; + let detail = match result { + Ended::Done => String::new(), + _ => last_error(log, start).unwrap_or_else(|| match output.status.code() { + Some(code) => format!("exit code {code}"), + None => "stopped by a signal".to_owned(), + }), + }; + Ran { + result, + counts, + detail, + } +} + +fn open_log(path: &Path) -> std::io::Result<(File, u64)> { + let mut options = OpenOptions::new(); + options.create(true).append(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let file = options.open(path)?; + let start = file.metadata()?.len(); + Ok((file, start)) +} + +/// The last `error:` line this run wrote to the log. +fn last_error(log: &Path, start: u64) -> Option { + let mut file = File::open(log).ok()?; + file.seek(SeekFrom::Start(start)).ok()?; + let mut text = String::new(); + file.read_to_string(&mut text).ok()?; + text.lines() + .rev() + .find_map(|l| l.strip_prefix("error: ")) + .map(str::to_owned) +} + +/// The per-type lines a step prints, summed: +/// `email: created=1 fetched=2 updated=0 deleted=0 skipped=0 failed=0`. +fn counts_of(stdout: &str) -> Counts { + let mut c = Counts::default(); + for line in stdout.lines() { + let Some((_, rest)) = line.split_once(": ") else { + continue; + }; + for field in rest.split_whitespace() { + let Some((key, n)) = field.split_once('=') else { + continue; + }; + let Ok(n) = n.parse::() else { continue }; + match key { + "fetched" => c.fetched += n, + "created" => c.created += n, + "updated" => c.updated += n, + "deleted" => c.deleted += n, + "skipped" => c.skipped += n, + "failed" => c.failed += n, + _ => {} + } + } + } + c +} + +/// `report.csv` in the work directory (`dry-run-report.csv` for a dry +/// run): one row per step per account, setup first. +fn write_report( + work: &Path, + config: &MigrateConfig, + setup: &[StepReport], + accounts: &[(String, Vec)], +) -> Result { + let path = work.join(if config.dry_run { + "dry-run-report.csv" + } else { + "report.csv" + }); + let mut out = + String::from("account,step,result,fetched,created,updated,deleted,skipped,failed,detail\n"); + let rows = setup.iter().map(|s| ("(setup)", s)).chain( + accounts + .iter() + .flat_map(|(email, steps)| steps.iter().map(move |s| (email.as_str(), s))), + ); + for (account, s) in rows { + let c = s.counts; + out.push_str(&format!( + "{account},{},{},{},{},{},{},{},{},{}\n", + s.step, + s.result, + c.fetched, + c.created, + c.updated, + c.deleted, + c.skipped, + c.failed, + csv_field(&s.detail) + )); + } + fs::write(&path, out)?; + Ok(path) +} + +fn csv_field(s: &str) -> String { + if s.contains([',', '"', '\n', '\r']) { + format!("\"{}\"", s.replace('"', "\"\"")) + } else { + s.to_owned() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn counts_are_summed_over_the_types() { + let c = counts_of( + "email: created=3 fetched=4 updated=1 deleted=0 skipped=2 failed=1\n\ + mailbox: created=2 fetched=2 updated=0 deleted=0 skipped=0 failed=0\n", + ); + assert_eq!((c.fetched, c.created, c.updated), (6, 5, 1)); + assert_eq!((c.skipped, c.failed), (2, 1)); + } + + #[test] + fn details_are_quoted_when_they_need_it() { + assert_eq!(csv_field("plain"), "plain"); + assert_eq!(csv_field("a, \"b\""), "\"a, \"\"b\"\"\""); + } +} diff --git a/src/migrate/plan.rs b/src/migrate/plan.rs new file mode 100644 index 0000000..7270ec8 --- /dev/null +++ b/src/migrate/plan.rs @@ -0,0 +1,430 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! The plan file: the steps to run once, the steps to run for each account, +//! and the accounts. A step is one inbuxa-migrate command line, written as +//! the docs write it, with `{email}`, `{local}`, `{domain}` and `{archive}` +//! filled in per account. Secrets are never in the plan: a step names the +//! environment variable that holds each one. + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; + +use serde::Deserialize; +use serde_json::Value; + +use crate::error::Error; + +pub const FORMAT: &str = "inbuxa-migrate-plan"; +pub const FORMAT_VERSION: u32 = 1; + +/// The variables a child reads its secrets from. All are cleared before a +/// step runs, so a step only ever sees the secrets it names. +pub const SECRET_VARS: &[&str] = &[ + "INBUXA_MIGRATE_PASSWORD", + "INBUXA_MIGRATE_API_KEY", + "INBUXA_MIGRATE_TOKEN", + "INBUXA_MIGRATE_EWS_CLIENT_SECRET", + "INBUXA_MIGRATE_GRAPH_TOKEN", +]; + +const ACCOUNT_COMMANDS: &[&str] = &[ + "import", + "export", + "export-imap", + "export-managesieve", + "export-caldav", + "export-carddav", + "export-files", +]; +const SETUP_COMMANDS: &[&str] = &["import-directory", "export-directory"]; +const PLACEHOLDERS: &[&str] = &["{email}", "{local}", "{domain}", "{archive}"]; + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct RawPlan { + format: String, + format_version: u32, + #[serde(default)] + setup: Vec, + accounts: RawAccounts, + steps: Vec, +} + +#[derive(Deserialize)] +#[serde(untagged)] +enum RawAccounts { + List(Vec), + Directory(FromDirectory), +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct FromDirectory { + directory: PathBuf, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct RawStep { + name: String, + args: Vec, + #[serde(default)] + secrets: Secrets, +} + +/// Each secret by the variable that holds it. +#[derive(Deserialize, Default)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Secrets { + password: Option, + token: Option, + client_secret: Option, + graph_token: Option, +} + +pub struct Plan { + /// The plan's own directory: steps run there, so relative paths in a + /// step mean what they say in the plan. + pub base: PathBuf, + pub setup: Vec, + pub accounts: Vec, + pub steps: Vec, +} + +/// No `Debug`: `env` holds secrets. +pub struct Step { + pub name: String, + pub args: Vec, + /// The child's secret variables, with their values. + pub env: Vec<(&'static str, String)>, +} + +impl Step { + /// The command line for one account, or for setup. + pub fn args_for(&self, account: Option<(&str, &Path)>) -> Vec { + let Some((email, archive)) = account else { + return self.args.clone(); + }; + let (local, domain) = email.rsplit_once('@').unwrap_or((email, "")); + let archive = archive.to_string_lossy(); + self.args + .iter() + .map(|a| { + a.replace("{email}", email) + .replace("{local}", local) + .replace("{domain}", domain) + .replace("{archive}", &archive) + }) + .collect() + } +} + +pub fn load(path: &Path) -> Result { + let text = std::fs::read_to_string(path) + .map_err(|e| Error::Usage(format!("can't read plan {}: {e}", path.display())))?; + let raw: RawPlan = serde_json::from_str(&text) + .map_err(|e| Error::Usage(format!("plan {}: {e}", path.display())))?; + if raw.format != FORMAT { + return Err(Error::Usage(format!( + "plan {}: format is \"{}\", not \"{FORMAT}\"", + path.display(), + raw.format + ))); + } + if raw.format_version != FORMAT_VERSION { + return Err(Error::Usage(format!( + "plan {}: formatVersion {} is not one this release reads ({FORMAT_VERSION})", + path.display(), + raw.format_version + ))); + } + let base = std::path::absolute(path)? + .parent() + .map(Path::to_path_buf) + .unwrap_or_default(); + + if raw.steps.is_empty() { + return Err(Error::Usage("the plan has no steps".to_owned())); + } + let mut names = HashSet::new(); + let mut setup = Vec::new(); + for step in raw.setup { + setup.push(step_of(step, false, &mut names)?); + } + let mut steps = Vec::new(); + for step in raw.steps { + steps.push(step_of(step, true, &mut names)?); + } + + let listed = match raw.accounts { + RawAccounts::List(list) => list, + RawAccounts::Directory(from) => directory_users(&base.join(from.directory))?, + }; + let mut accounts = Vec::new(); + let mut seen = HashSet::new(); + for email in listed { + let email = account_email(&email)?; + if !seen.insert(email.clone()) { + return Err(Error::Usage(format!("account {email} is listed twice"))); + } + accounts.push(email); + } + if accounts.is_empty() { + return Err(Error::Usage("the plan lists no accounts".to_owned())); + } + Ok(Plan { + base, + setup, + accounts, + steps, + }) +} + +fn step_of(raw: RawStep, per_account: bool, names: &mut HashSet) -> Result { + let name = raw.name; + if name.is_empty() + || !name + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') + { + return Err(Error::Usage(format!( + "step name \"{name}\": use letters, digits, - and _" + ))); + } + if !names.insert(name.clone()) { + return Err(Error::Usage(format!("step name \"{name}\" is used twice"))); + } + check_args(&name, &raw.args, per_account)?; + + let mut env = Vec::new(); + let secrets = raw.secrets; + for (child_var, from) in [ + ("INBUXA_MIGRATE_PASSWORD", secrets.password), + ("INBUXA_MIGRATE_API_KEY", secrets.token), + ("INBUXA_MIGRATE_EWS_CLIENT_SECRET", secrets.client_secret), + ("INBUXA_MIGRATE_GRAPH_TOKEN", secrets.graph_token), + ] { + let Some(from) = from else { continue }; + match std::env::var(&from) { + Ok(value) if !value.is_empty() => env.push((child_var, value)), + _ => { + return Err(Error::Usage(format!( + "step {name}: ${from} is not set; it holds a secret the step names" + ))); + } + } + } + Ok(Step { + name, + args: raw.args, + env, + }) +} + +fn check_args(name: &str, args: &[String], per_account: bool) -> Result<(), Error> { + let usage = |m: String| Err(Error::Usage(format!("step {name}: {m}"))); + let (allowed, kind) = if per_account { + (ACCOUNT_COMMANDS, "an account step") + } else { + (SETUP_COMMANDS, "a setup step") + }; + match args.first() { + Some(command) if allowed.contains(&command.as_str()) => {} + Some(command) => { + return usage(format!( + "{kind} runs one of {}, not \"{command}\"", + allowed.join(", ") + )); + } + None => return usage("args is empty".to_owned()), + } + for (i, arg) in args.iter().enumerate() { + let flag = arg.split_once('=').map_or(arg.as_str(), |(f, _)| f); + match flag { + "--auth-password" | "--ews-client-secret" => { + return usage(format!( + "{flag} would put a secret in the plan; name its variable in \"secrets\"" + )); + } + "--auth-bearer" | "--access-token" => { + let next = args.get(i + 1); + if arg.contains('=') || next.is_some_and(|n| !n.starts_with('-')) { + return usage(format!( + "{flag} takes its token from \"secrets\"; write it with no value, \ + before another flag" + )); + } + } + "--dry-run" => { + return usage("leave out --dry-run; migrate --dry-run adds it".to_owned()); + } + _ => {} + } + } + let placeholders = args + .iter() + .any(|a| PLACEHOLDERS.iter().any(|p| a.contains(p))); + if per_account && !args.iter().any(|a| a.contains("{archive}")) { + return usage("an account step has to name {archive}".to_owned()); + } + if !per_account && placeholders { + return usage( + "a setup step runs once, so it can't use {email}, {archive} and the rest".to_owned(), + ); + } + Ok(()) +} + +/// The users in a `directory.json` that `export-directory` wrote. +fn directory_users(path: &Path) -> Result, Error> { + let text = std::fs::read_to_string(path) + .map_err(|e| Error::Usage(format!("can't read {}: {e}", path.display())))?; + let doc: Value = serde_json::from_str(&text) + .map_err(|e| Error::Usage(format!("{}: {e}", path.display())))?; + if doc.get("format").and_then(Value::as_str) != Some("inbuxa-migrate-directory") { + return Err(Error::Usage(format!( + "{} is not a directory.json from export-directory", + path.display() + ))); + } + Ok(doc + .get("users") + .and_then(Value::as_array) + .map(|users| { + users + .iter() + .filter_map(|u| u.get("email").and_then(Value::as_str)) + .map(str::to_owned) + .collect() + }) + .unwrap_or_default()) +} + +/// An address that is safe as a directory name, lowercased. +fn account_email(email: &str) -> Result { + let email = email.trim().to_ascii_lowercase(); + let ok = match email.split_once('@') { + Some((local, domain)) => { + !local.is_empty() + && !domain.is_empty() + && !domain.contains('@') + && !email.starts_with('.') + && !email + .chars() + .any(|c| c.is_whitespace() || c.is_control() || matches!(c, '/' | '\\')) + } + None => false, + }; + if !ok { + return Err(Error::Usage(format!( + "account \"{email}\" is not an address the plan can use" + ))); + } + Ok(email) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn args(list: &[&str]) -> Vec { + list.iter().map(|s| (*s).to_owned()).collect() + } + + fn err(list: &[&str], per_account: bool) -> String { + check_args("s", &args(list), per_account) + .unwrap_err() + .to_string() + } + + #[test] + fn secrets_and_dry_run_stay_out_of_the_args() { + let e = err( + &["import", "imap", "--auth-password", "x", "{archive}"], + true, + ); + assert!(e.contains("secrets"), "{e}"); + let e = err(&["import", "imap", "--auth-password=x", "{archive}"], true); + assert!(e.contains("secrets"), "{e}"); + // A bare --auth-bearer before the archive would take it as the token. + let e = err(&["export", "--auth-bearer", "{archive}"], true); + assert!(e.contains("before another flag"), "{e}"); + let e = err(&["export", "--dry-run", "{archive}"], true); + assert!(e.contains("--dry-run"), "{e}"); + check_args( + "s", + &args(&[ + "export", + "--auth-bearer", + "--account-name", + "{email}", + "{archive}", + ]), + true, + ) + .unwrap(); + } + + #[test] + fn steps_run_the_commands_they_are_for() { + let e = err(&["import-directory", "dir"], true); + assert!(e.contains("an account step runs one of"), "{e}"); + let e = err(&["import", "imap", "--url", "x"], true); + assert!(e.contains("{archive}"), "{e}"); + let e = err(&["import-directory", "--url", "{domain}", "dir"], false); + assert!(e.contains("runs once"), "{e}"); + check_args("s", &args(&["import-directory", "dir"]), false).unwrap(); + } + + #[test] + fn placeholders_are_filled_per_account() { + let step = Step { + name: "mail".into(), + args: args(&[ + "import", + "imap", + "--auth-basic", + "{local}*master@{domain}", + "{archive}", + "^A\\d{4}$", + ]), + env: Vec::new(), + }; + assert_eq!( + step.args_for(Some(( + "ann@example.org", + Path::new("/w/ann/archive.sqlite") + ))), + args(&[ + "import", + "imap", + "--auth-basic", + "ann*master@example.org", + "/w/ann/archive.sqlite", + "^A\\d{4}$" + ]) + ); + } + + #[test] + fn accounts_are_plain_addresses() { + assert_eq!( + account_email(" Ann@Example.org ").unwrap(), + "ann@example.org" + ); + for bad in [ + "ann", + "@example.org", + "ann@", + "a/b@example.org", + "../x@y", + "a b@c", + ] { + assert!(account_email(bad).is_err(), "{bad}"); + } + } +} diff --git a/tests/mock_migrate.rs b/tests/mock_migrate.rs new file mode 100644 index 0000000..92a458a --- /dev/null +++ b/tests/mock_migrate.rs @@ -0,0 +1,180 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +//! `migrate` runs a plan's steps for each account as runs of the real +//! binary: Maildir in, files out, offline. One account fails and the rest +//! go on; a second run resumes only what didn't finish; `--again` runs it +//! all once more. Counts in the report are summed over the types: a +//! mailbox and an email are two. + +use std::fs; +use std::path::{Path, PathBuf}; + +use inbuxa_migrate::logging::Logger; +use inbuxa_migrate::migrate::{self, MigrateConfig, Outcome}; +use rusqlite::Connection; +use serde_json::json; +use tempfile::TempDir; + +fn message(dir: &Path, name: &str, subject: &str) { + for sub in ["cur", "new", "tmp"] { + fs::create_dir_all(dir.join(sub)).unwrap(); + } + fs::write( + dir.join("cur").join(format!("{name}:2,S")), + format!( + "From: a@example.org\r\nTo: b@example.org\r\nSubject: {subject}\r\n\ + Message-ID: <{name}@example.org>\r\nDate: Mon, 4 Mar 2024 09:00:00 +0000\r\n\r\nHi\r\n" + ), + ) + .unwrap(); +} + +fn config(plan: &Path, work: &Path) -> MigrateConfig { + MigrateConfig { + plan: plan.to_path_buf(), + work: work.to_path_buf(), + parallel: 2, + again: false, + only: Vec::new(), + dry_run: false, + program: PathBuf::from(env!("CARGO_BIN_EXE_inbuxa-migrate")), + logger: Logger::from_flags(true, 0), + } +} + +fn emails(archive: &Path) -> i64 { + Connection::open(archive) + .unwrap() + .query_row("SELECT COUNT(*) FROM emails", [], |r| r.get(0)) + .unwrap() +} + +fn report(outcome: &Outcome) -> Vec { + fs::read_to_string(&outcome.report) + .unwrap() + .lines() + .skip(1) + .map(|l| l.split(',').take(5).collect::>().join(",")) + .collect() +} + +fn write_plan(path: &Path, steps: serde_json::Value) { + let plan = json!({ + "format": "inbuxa-migrate-plan", + "formatVersion": 1, + "accounts": ["ann@example.org", "Bob@example.org"], + "steps": steps, + }); + fs::write(path, serde_json::to_vec_pretty(&plan).unwrap()).unwrap(); +} + +#[test] +fn accounts_run_their_steps_resume_and_run_again() { + let dir = TempDir::new().unwrap(); + let root = dir.path(); + message(&root.join("mail/ann"), "m1", "First"); + let plan = root.join("plan.json"); + // Relative paths are the plan's own: the steps run in its directory. + write_plan( + &plan, + json!([ + {"name": "mail", "args": ["import", "maildir", "mail/{local}", "{archive}"]}, + {"name": "files", "args": ["export-files", "{archive}", "out/{email}"]}, + ]), + ); + let work = root.join("work"); + + // Bob has no Maildir yet: his import fails, his export never runs. + let first = migrate::run(config(&plan, &work)).unwrap(); + assert_eq!((first.done, first.unfinished), (1, 1)); + assert_eq!( + report(&first), + [ + "ann@example.org,mail,done,2,2", + "ann@example.org,files,done,0,2", + "bob@example.org,mail,failed,0,0", + "bob@example.org,files,not run,0,0", + ] + ); + let detail = fs::read_to_string(&first.report).unwrap(); + assert!( + detail.contains("mail/bob"), + "the reason is reported: {detail}" + ); + assert_eq!(emails(&work.join("ann@example.org/archive.sqlite")), 1); + assert!(root.join("out/ann@example.org/manifest.json").exists()); + let log = fs::read_to_string(work.join("ann@example.org/mail.log")).unwrap(); + assert!( + log.starts_with("== inbuxa-migrate import maildir mail/ann "), + "{log}" + ); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = |p: &Path| fs::metadata(p).unwrap().permissions().mode() & 0o777; + assert_eq!(mode(&work), 0o700); + assert_eq!(mode(&work.join("ann@example.org/mail.log")), 0o600); + } + + // The second run leaves Ann alone (her export-files would refuse a + // directory that isn't empty) and finishes Bob. + message(&root.join("mail/bob"), "m2", "Second"); + let second = migrate::run(config(&plan, &work)).unwrap(); + assert_eq!((second.done, second.unfinished), (2, 0)); + assert_eq!( + report(&second), + [ + "ann@example.org,mail,done earlier,0,0", + "ann@example.org,files,done earlier,0,0", + "bob@example.org,mail,done,2,2", + "bob@example.org,files,done,0,2", + ] + ); + + // --again: every step runs once more and picks up what's new. + write_plan( + &plan, + json!([{"name": "mail", "args": ["import", "maildir", "mail/{local}", "{archive}"]}]), + ); + message(&root.join("mail/ann"), "m3", "Third"); + let mut again = config(&plan, &work); + again.again = true; + again.only = vec!["ann@example.org".into()]; + let third = migrate::run(again).unwrap(); + assert_eq!(report(&third), ["ann@example.org,mail,done,1,1"]); + assert_eq!(emails(&work.join("ann@example.org/archive.sqlite")), 2); +} + +#[test] +fn a_step_sees_only_the_secrets_it_names() { + let dir = TempDir::new().unwrap(); + let plan = dir.path().join("plan.json"); + let step = |secrets: serde_json::Value| { + json!([{ + "name": "mail", + "args": ["import", "imap", "--url", "imap://127.0.0.1:9", "--allow-cleartext", + "--auth-basic", "{email}", "{archive}"], + "secrets": secrets, + }]) + }; + + write_plan(&plan, step(json!({"password": "MIGRATE_TEST_UNSET_VAR"}))); + let err = migrate::run(config(&plan, &dir.path().join("w1"))) + .err() + .unwrap() + .to_string(); + assert!(err.contains("$MIGRATE_TEST_UNSET_VAR is not set"), "{err}"); + + // No secret named: the password this test run may have in its own + // environment is not passed on, and with no terminal the step fails + // at once rather than waiting for one. + write_plan(&plan, step(json!({}))); + let out = migrate::run(config(&plan, &dir.path().join("w2"))).unwrap(); + assert_eq!(out.unfinished, 2); + let text = fs::read_to_string(&out.report).unwrap(); + assert!(text.contains("no password provided"), "{text}"); +} -- 2.54.0