migrate --progress json #51

Merged
jcoffey-dev merged 1 commits from feat/migrate-progress-json into main 2026-10-31 00:48:53 +00:00
6 changed files with 160 additions and 10 deletions

No files matched your search

+5
View File
@@ -7,6 +7,11 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a
## [Unreleased]
### Added
- `migrate --progress json` writes one JSON event per line on stdout (a
step begins or ends, an account ends, the run ends), for a program that
runs a migration and shows its progress.
## [27.1-rc.7]
The seventh release candidate of the 27.1 series: upstream's 1.0.12 and
+8 -1
View File
@@ -603,7 +603,7 @@ server and counts what it would create, sends nothing and writes no file;
```
inbuxa-migrate migrate --plan <FILE> [--parallel <N>] [--again]
[--account <EMAIL>]... [--dry-run] [-q] <DIR>
[--account <EMAIL>]... [--dry-run] [-q] [--progress json] <DIR>
```
Moves a list of accounts with one command. The plan file names the steps
@@ -674,6 +674,13 @@ account, the state a later run resumes from, and the report.
`done earlier`), the counts summed over the step's types, and, for one
that didn't finish, its last error. Exit code 5 means some account
didn't finish.
- **Progress for another program.** `--progress json` writes one JSON
object per line on stdout, and nothing else there. `event` says which:
`start` (`accounts`, `parallel`, `dryRun`), `setup` (a setup step ended),
`step` (an account's step began, or ended with `result`, `counts` and
`detail`), `account` (an account ended: `email`, `n` of `total`,
`result`) and `end` (`done`, `unfinished`, `report`). Messages still go
to stderr.
The work directory is made readable by its owner only, and so are the
logs. With `--dry-run`, each step runs with `--dry-run`, nothing is
+9
View File
@@ -738,6 +738,14 @@ struct MigrateArgs {
#[arg(short = 'q', long, help = "Quiet: no line per account")]
quiet: bool,
#[arg(
long,
value_name = "FORMAT",
value_parser = ["json"],
help = "Progress for another program: `json` writes an event per line on stdout, and nothing else"
)]
progress: Option<String>,
#[arg(
value_name = "DIR",
help = "Work directory: an archive and logs per account, the state and the report"
@@ -1468,6 +1476,7 @@ fn resolve_migrate(args: MigrateArgs) -> Result<Action, Error> {
dry_run: args.dry_run,
program: std::env::current_exe()?,
logger: Logger::from_flags(args.quiet, 0),
progress_json: args.progress.is_some(),
}))
}
+9 -6
View File
@@ -126,14 +126,17 @@ fn run() -> i32 {
)
}
Action::Migrate(config) => {
let json = config.progress_json;
return match migrate::run(config) {
Ok(outcome) => {
println!(
"accounts: {} done, {} not finished; report: {}",
outcome.done,
outcome.unfinished,
outcome.report.display()
);
if !json {
println!(
"accounts: {} done, {} not finished; report: {}",
outcome.done,
outcome.unfinished,
outcome.report.display()
);
}
if outcome.unfinished > 0 {
eprintln!(
"error: some accounts did not finish; the report says why. Fix that \
+67 -3
View File
@@ -21,6 +21,7 @@ use std::sync::Mutex;
use std::sync::atomic::{AtomicUsize, Ordering};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use crate::error::Error;
use crate::logging::Logger;
@@ -40,6 +41,27 @@ pub struct MigrateConfig {
/// The binary each step runs: this one.
pub program: PathBuf,
pub logger: Logger,
/// Progress as JSON lines on stdout, for a program that runs this one.
pub progress_json: bool,
}
/// Writes one progress event, a JSON object on a line of its own, when
/// `--progress json` asked for them. The events, by `event`:
///
/// - `start`: `accounts`, `parallel`, `dryRun`
/// - `setup`: a setup step ended: `step`, `result`, `counts`, `detail`
/// - `step`: an account's step began (`result` absent) or ended
/// - `account`: an account ended: `email`, `n` of `total`, `result`
/// - `end`: `done`, `unfinished`, `report`
///
/// `result` is `done`, `partial`, `failed`, `done earlier` or `not run`. A
/// closed stdout is ignored, as the guide's output is.
fn emit(config: &MigrateConfig, event: &str, mut fields: Value) {
if !config.progress_json {
return;
}
fields["event"] = json!(event);
let _ = writeln!(std::io::stdout().lock(), "{fields}");
}
pub struct Outcome {
@@ -76,7 +98,7 @@ struct StepReport {
detail: String,
}
#[derive(Default, Clone, Copy)]
#[derive(Default, Clone, Copy, Serialize)]
struct Counts {
fetched: u64,
created: u64,
@@ -101,6 +123,11 @@ pub fn run(config: MigrateConfig) -> Result<Outcome, Error> {
let work = std::path::absolute(&config.work)?;
private_dir(&work)?;
emit(
&config,
"start",
json!({"accounts": accounts.len(), "parallel": config.parallel, "dryRun": config.dry_run}),
);
let state_path = work.join("state.json");
let state = Mutex::new(load_state(&state_path)?);
let save = |state: &State| -> Result<(), Error> {
@@ -126,7 +153,9 @@ pub fn run(config: MigrateConfig) -> Result<Outcome, Error> {
}
let log = work.join("setup").join(format!("{}.log", step.name));
let ran = run_step(&config, &plan, step, None, &log);
setup_reports.push(report_of(&step.name, &ran));
let report = report_of(&step.name, &ran);
emit(&config, "setup", report.json());
setup_reports.push(report);
if !config.dry_run {
lock(&state).setup.insert(step.name.clone(), ran.result);
}
@@ -156,6 +185,12 @@ pub fn run(config: MigrateConfig) -> Result<Outcome, Error> {
match migrate_account(&config, &plan, &work, email, &state, &save) {
Ok(steps) => {
let n = finished.fetch_add(1, Ordering::SeqCst) + 1;
emit(
&config,
"account",
json!({"email": email, "n": n, "total": total,
"result": result_of(&steps)}),
);
if config.logger.enabled(crate::logging::LEVEL_DEFAULT) {
eprintln!("[{n}/{total}] {email}: {}", line_of(&steps));
}
@@ -187,6 +222,12 @@ pub fn run(config: MigrateConfig) -> Result<Outcome, Error> {
})
.count();
let report = write_report(&work, &config, &setup_reports, &reports)?;
emit(
&config,
"end",
json!({"done": reports.len() - unfinished, "unfinished": unfinished,
"report": report.display().to_string()}),
);
Ok(Outcome {
done: reports.len() - unfinished,
unfinished,
@@ -299,8 +340,13 @@ fn migrate_account(
}
running = true;
let log = dir.join(format!("{}.log", step.name));
emit(config, "step", json!({"email": email, "step": step.name}));
let ran = run_step(config, plan, step, Some((email, &archive)), &log);
reports.push(report_of(&step.name, &ran));
let report = report_of(&step.name, &ran);
let mut ended = report.json();
ended["email"] = json!(email);
emit(config, "step", ended);
reports.push(report);
if !config.dry_run {
let mut s = lock(state);
s.accounts
@@ -336,6 +382,24 @@ fn report_of(step: &str, ran: &Ran) -> StepReport {
}
}
impl StepReport {
fn json(&self) -> Value {
json!({
"step": self.step, "result": self.result,
"counts": self.counts, "detail": self.detail,
})
}
}
/// An account's result: the first step that didn't finish decides it.
fn result_of(steps: &[StepReport]) -> &'static str {
steps
.iter()
.map(|s| s.result)
.find(|r| *r == "failed" || *r == "partial")
.unwrap_or("done")
}
fn line_of(steps: &[StepReport]) -> String {
match steps
.iter()
+62
View File
@@ -43,6 +43,7 @@ fn config(plan: &Path, work: &Path) -> MigrateConfig {
dry_run: false,
program: PathBuf::from(env!("CARGO_BIN_EXE_inbuxa-migrate")),
logger: Logger::from_flags(true, 0),
progress_json: false,
}
}
@@ -178,3 +179,64 @@ fn a_step_sees_only_the_secrets_it_names() {
let text = fs::read_to_string(&out.report).unwrap();
assert!(text.contains("no password provided"), "{text}");
}
#[test]
fn progress_json_is_one_event_per_line_and_nothing_else() {
let dir = TempDir::new().unwrap();
let root = dir.path();
message(&root.join("mail/ann"), "m1", "First");
let plan = root.join("plan.json");
write_plan(
&plan,
json!([{"name": "mail", "args": ["import", "maildir", "mail/{local}", "{archive}"]}]),
);
let out = std::process::Command::new(env!("CARGO_BIN_EXE_inbuxa-migrate"))
.args(["migrate", "--parallel", "1", "--progress", "json", "--plan"])
.arg(&plan)
.arg(root.join("work"))
.output()
.unwrap();
// Bob has no Maildir: the run ends unfinished.
assert_eq!(out.status.code(), Some(5));
let events: Vec<serde_json::Value> = String::from_utf8(out.stdout)
.unwrap()
.lines()
.map(|l| serde_json::from_str(l).unwrap_or_else(|e| panic!("{e}: {l}")))
.collect();
let kinds: Vec<&str> = events
.iter()
.map(|e| e["event"].as_str().unwrap())
.collect();
assert_eq!(
kinds,
[
"start", "step", "step", "account", "step", "step", "account", "end"
]
);
assert_eq!(events[0]["accounts"], 2);
assert_eq!(
events[1],
json!({"event": "step", "email": "[email protected]", "step": "mail"})
);
assert_eq!(events[2]["result"], "done");
assert_eq!(events[2]["counts"]["created"], 2);
assert_eq!(
events[3],
json!({"event": "account", "email": "[email protected]", "n": 1, "total": 2, "result": "done"})
);
assert_eq!(events[5]["result"], "failed");
assert!(
events[5]["detail"].as_str().unwrap().contains("mail/bob"),
"{}",
events[5]
);
assert_eq!(events[6]["result"], "failed");
assert_eq!(events[7]["done"], 1);
assert_eq!(events[7]["unfinished"], 1);
assert!(
events[7]["report"]
.as_str()
.unwrap()
.ends_with("report.csv")
);
}