diff --git a/CHANGELOG.md b/CHANGELOG.md index deb0223..7c99e61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a ## [Unreleased] +### Added +- `migrate --progress json` writes one JSON event per line on stdout (a + step begins or ends, an account ends, the run ends), for a program that + runs a migration and shows its progress. + ## [27.1-rc.7] The seventh release candidate of the 27.1 series: upstream's 1.0.12 and diff --git a/docs/usage.md b/docs/usage.md index 2a83d37..a9c67de 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -603,7 +603,7 @@ server and counts what it would create, sends nothing and writes no file; ``` inbuxa-migrate migrate --plan [--parallel ] [--again] - [--account ]... [--dry-run] [-q] + [--account ]... [--dry-run] [-q] [--progress json] ``` Moves a list of accounts with one command. The plan file names the steps @@ -674,6 +674,13 @@ account, the state a later run resumes from, and the report. `done earlier`), the counts summed over the step's types, and, for one that didn't finish, its last error. Exit code 5 means some account didn't finish. +- **Progress for another program.** `--progress json` writes one JSON + object per line on stdout, and nothing else there. `event` says which: + `start` (`accounts`, `parallel`, `dryRun`), `setup` (a setup step ended), + `step` (an account's step began, or ended with `result`, `counts` and + `detail`), `account` (an account ended: `email`, `n` of `total`, + `result`) and `end` (`done`, `unfinished`, `report`). Messages still go + to stderr. The work directory is made readable by its owner only, and so are the logs. With `--dry-run`, each step runs with `--dry-run`, nothing is diff --git a/src/cli.rs b/src/cli.rs index 25425e9..7ec00a9 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -738,6 +738,14 @@ struct MigrateArgs { #[arg(short = 'q', long, help = "Quiet: no line per account")] quiet: bool, + #[arg( + long, + value_name = "FORMAT", + value_parser = ["json"], + help = "Progress for another program: `json` writes an event per line on stdout, and nothing else" + )] + progress: Option, + #[arg( value_name = "DIR", help = "Work directory: an archive and logs per account, the state and the report" @@ -1468,6 +1476,7 @@ fn resolve_migrate(args: MigrateArgs) -> Result { dry_run: args.dry_run, program: std::env::current_exe()?, logger: Logger::from_flags(args.quiet, 0), + progress_json: args.progress.is_some(), })) } diff --git a/src/main.rs b/src/main.rs index 5fec816..1a94eb5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -126,14 +126,17 @@ fn run() -> i32 { ) } Action::Migrate(config) => { + let json = config.progress_json; return match migrate::run(config) { Ok(outcome) => { - println!( - "accounts: {} done, {} not finished; report: {}", - outcome.done, - outcome.unfinished, - outcome.report.display() - ); + if !json { + println!( + "accounts: {} done, {} not finished; report: {}", + outcome.done, + outcome.unfinished, + outcome.report.display() + ); + } if outcome.unfinished > 0 { eprintln!( "error: some accounts did not finish; the report says why. Fix that \ diff --git a/src/migrate/mod.rs b/src/migrate/mod.rs index 90eecb9..5709353 100644 --- a/src/migrate/mod.rs +++ b/src/migrate/mod.rs @@ -21,6 +21,7 @@ use std::sync::Mutex; use std::sync::atomic::{AtomicUsize, Ordering}; use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; use crate::error::Error; use crate::logging::Logger; @@ -40,6 +41,27 @@ pub struct MigrateConfig { /// The binary each step runs: this one. pub program: PathBuf, pub logger: Logger, + /// Progress as JSON lines on stdout, for a program that runs this one. + pub progress_json: bool, +} + +/// Writes one progress event, a JSON object on a line of its own, when +/// `--progress json` asked for them. The events, by `event`: +/// +/// - `start`: `accounts`, `parallel`, `dryRun` +/// - `setup`: a setup step ended: `step`, `result`, `counts`, `detail` +/// - `step`: an account's step began (`result` absent) or ended +/// - `account`: an account ended: `email`, `n` of `total`, `result` +/// - `end`: `done`, `unfinished`, `report` +/// +/// `result` is `done`, `partial`, `failed`, `done earlier` or `not run`. A +/// closed stdout is ignored, as the guide's output is. +fn emit(config: &MigrateConfig, event: &str, mut fields: Value) { + if !config.progress_json { + return; + } + fields["event"] = json!(event); + let _ = writeln!(std::io::stdout().lock(), "{fields}"); } pub struct Outcome { @@ -76,7 +98,7 @@ struct StepReport { detail: String, } -#[derive(Default, Clone, Copy)] +#[derive(Default, Clone, Copy, Serialize)] struct Counts { fetched: u64, created: u64, @@ -101,6 +123,11 @@ pub fn run(config: MigrateConfig) -> Result { let work = std::path::absolute(&config.work)?; private_dir(&work)?; + emit( + &config, + "start", + json!({"accounts": accounts.len(), "parallel": config.parallel, "dryRun": config.dry_run}), + ); let state_path = work.join("state.json"); let state = Mutex::new(load_state(&state_path)?); let save = |state: &State| -> Result<(), Error> { @@ -126,7 +153,9 @@ pub fn run(config: MigrateConfig) -> Result { } let log = work.join("setup").join(format!("{}.log", step.name)); let ran = run_step(&config, &plan, step, None, &log); - setup_reports.push(report_of(&step.name, &ran)); + let report = report_of(&step.name, &ran); + emit(&config, "setup", report.json()); + setup_reports.push(report); if !config.dry_run { lock(&state).setup.insert(step.name.clone(), ran.result); } @@ -156,6 +185,12 @@ pub fn run(config: MigrateConfig) -> Result { match migrate_account(&config, &plan, &work, email, &state, &save) { Ok(steps) => { let n = finished.fetch_add(1, Ordering::SeqCst) + 1; + emit( + &config, + "account", + json!({"email": email, "n": n, "total": total, + "result": result_of(&steps)}), + ); if config.logger.enabled(crate::logging::LEVEL_DEFAULT) { eprintln!("[{n}/{total}] {email}: {}", line_of(&steps)); } @@ -187,6 +222,12 @@ pub fn run(config: MigrateConfig) -> Result { }) .count(); let report = write_report(&work, &config, &setup_reports, &reports)?; + emit( + &config, + "end", + json!({"done": reports.len() - unfinished, "unfinished": unfinished, + "report": report.display().to_string()}), + ); Ok(Outcome { done: reports.len() - unfinished, unfinished, @@ -299,8 +340,13 @@ fn migrate_account( } running = true; let log = dir.join(format!("{}.log", step.name)); + emit(config, "step", json!({"email": email, "step": step.name})); let ran = run_step(config, plan, step, Some((email, &archive)), &log); - reports.push(report_of(&step.name, &ran)); + let report = report_of(&step.name, &ran); + let mut ended = report.json(); + ended["email"] = json!(email); + emit(config, "step", ended); + reports.push(report); if !config.dry_run { let mut s = lock(state); s.accounts @@ -336,6 +382,24 @@ fn report_of(step: &str, ran: &Ran) -> StepReport { } } +impl StepReport { + fn json(&self) -> Value { + json!({ + "step": self.step, "result": self.result, + "counts": self.counts, "detail": self.detail, + }) + } +} + +/// An account's result: the first step that didn't finish decides it. +fn result_of(steps: &[StepReport]) -> &'static str { + steps + .iter() + .map(|s| s.result) + .find(|r| *r == "failed" || *r == "partial") + .unwrap_or("done") +} + fn line_of(steps: &[StepReport]) -> String { match steps .iter() diff --git a/tests/mock_migrate.rs b/tests/mock_migrate.rs index 92a458a..2ce9c34 100644 --- a/tests/mock_migrate.rs +++ b/tests/mock_migrate.rs @@ -43,6 +43,7 @@ fn config(plan: &Path, work: &Path) -> MigrateConfig { dry_run: false, program: PathBuf::from(env!("CARGO_BIN_EXE_inbuxa-migrate")), logger: Logger::from_flags(true, 0), + progress_json: false, } } @@ -178,3 +179,64 @@ fn a_step_sees_only_the_secrets_it_names() { let text = fs::read_to_string(&out.report).unwrap(); assert!(text.contains("no password provided"), "{text}"); } + +#[test] +fn progress_json_is_one_event_per_line_and_nothing_else() { + let dir = TempDir::new().unwrap(); + let root = dir.path(); + message(&root.join("mail/ann"), "m1", "First"); + let plan = root.join("plan.json"); + write_plan( + &plan, + json!([{"name": "mail", "args": ["import", "maildir", "mail/{local}", "{archive}"]}]), + ); + let out = std::process::Command::new(env!("CARGO_BIN_EXE_inbuxa-migrate")) + .args(["migrate", "--parallel", "1", "--progress", "json", "--plan"]) + .arg(&plan) + .arg(root.join("work")) + .output() + .unwrap(); + // Bob has no Maildir: the run ends unfinished. + assert_eq!(out.status.code(), Some(5)); + let events: Vec = String::from_utf8(out.stdout) + .unwrap() + .lines() + .map(|l| serde_json::from_str(l).unwrap_or_else(|e| panic!("{e}: {l}"))) + .collect(); + let kinds: Vec<&str> = events + .iter() + .map(|e| e["event"].as_str().unwrap()) + .collect(); + assert_eq!( + kinds, + [ + "start", "step", "step", "account", "step", "step", "account", "end" + ] + ); + assert_eq!(events[0]["accounts"], 2); + assert_eq!( + events[1], + json!({"event": "step", "email": "ann@example.org", "step": "mail"}) + ); + assert_eq!(events[2]["result"], "done"); + assert_eq!(events[2]["counts"]["created"], 2); + assert_eq!( + events[3], + json!({"event": "account", "email": "ann@example.org", "n": 1, "total": 2, "result": "done"}) + ); + assert_eq!(events[5]["result"], "failed"); + assert!( + events[5]["detail"].as_str().unwrap().contains("mail/bob"), + "{}", + events[5] + ); + assert_eq!(events[6]["result"], "failed"); + assert_eq!(events[7]["done"], 1); + assert_eq!(events[7]["unfinished"], 1); + assert!( + events[7]["report"] + .as_str() + .unwrap() + .ends_with("report.csv") + ); +}