migrate --progress json #51
No files matched your search
@@ -7,6 +7,11 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
- `migrate --progress json` writes one JSON event per line on stdout (a
|
||||
step begins or ends, an account ends, the run ends), for a program that
|
||||
runs a migration and shows its progress.
|
||||
|
||||
## [27.1-rc.7]
|
||||
|
||||
The seventh release candidate of the 27.1 series: upstream's 1.0.12 and
|
||||
|
||||
+8
-1
@@ -603,7 +603,7 @@ server and counts what it would create, sends nothing and writes no file;
|
||||
|
||||
```
|
||||
inbuxa-migrate migrate --plan <FILE> [--parallel <N>] [--again]
|
||||
[--account <EMAIL>]... [--dry-run] [-q] <DIR>
|
||||
[--account <EMAIL>]... [--dry-run] [-q] [--progress json] <DIR>
|
||||
```
|
||||
|
||||
Moves a list of accounts with one command. The plan file names the steps
|
||||
@@ -674,6 +674,13 @@ account, the state a later run resumes from, and the report.
|
||||
`done earlier`), the counts summed over the step's types, and, for one
|
||||
that didn't finish, its last error. Exit code 5 means some account
|
||||
didn't finish.
|
||||
- **Progress for another program.** `--progress json` writes one JSON
|
||||
object per line on stdout, and nothing else there. `event` says which:
|
||||
`start` (`accounts`, `parallel`, `dryRun`), `setup` (a setup step ended),
|
||||
`step` (an account's step began, or ended with `result`, `counts` and
|
||||
`detail`), `account` (an account ended: `email`, `n` of `total`,
|
||||
`result`) and `end` (`done`, `unfinished`, `report`). Messages still go
|
||||
to stderr.
|
||||
|
||||
The work directory is made readable by its owner only, and so are the
|
||||
logs. With `--dry-run`, each step runs with `--dry-run`, nothing is
|
||||
|
||||
@@ -738,6 +738,14 @@ struct MigrateArgs {
|
||||
#[arg(short = 'q', long, help = "Quiet: no line per account")]
|
||||
quiet: bool,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
value_name = "FORMAT",
|
||||
value_parser = ["json"],
|
||||
help = "Progress for another program: `json` writes an event per line on stdout, and nothing else"
|
||||
)]
|
||||
progress: Option<String>,
|
||||
|
||||
#[arg(
|
||||
value_name = "DIR",
|
||||
help = "Work directory: an archive and logs per account, the state and the report"
|
||||
@@ -1468,6 +1476,7 @@ fn resolve_migrate(args: MigrateArgs) -> Result<Action, Error> {
|
||||
dry_run: args.dry_run,
|
||||
program: std::env::current_exe()?,
|
||||
logger: Logger::from_flags(args.quiet, 0),
|
||||
progress_json: args.progress.is_some(),
|
||||
}))
|
||||
}
|
||||
|
||||
|
||||
+9
-6
@@ -126,14 +126,17 @@ fn run() -> i32 {
|
||||
)
|
||||
}
|
||||
Action::Migrate(config) => {
|
||||
let json = config.progress_json;
|
||||
return match migrate::run(config) {
|
||||
Ok(outcome) => {
|
||||
println!(
|
||||
"accounts: {} done, {} not finished; report: {}",
|
||||
outcome.done,
|
||||
outcome.unfinished,
|
||||
outcome.report.display()
|
||||
);
|
||||
if !json {
|
||||
println!(
|
||||
"accounts: {} done, {} not finished; report: {}",
|
||||
outcome.done,
|
||||
outcome.unfinished,
|
||||
outcome.report.display()
|
||||
);
|
||||
}
|
||||
if outcome.unfinished > 0 {
|
||||
eprintln!(
|
||||
"error: some accounts did not finish; the report says why. Fix that \
|
||||
|
||||
+67
-3
@@ -21,6 +21,7 @@ use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::error::Error;
|
||||
use crate::logging::Logger;
|
||||
@@ -40,6 +41,27 @@ pub struct MigrateConfig {
|
||||
/// The binary each step runs: this one.
|
||||
pub program: PathBuf,
|
||||
pub logger: Logger,
|
||||
/// Progress as JSON lines on stdout, for a program that runs this one.
|
||||
pub progress_json: bool,
|
||||
}
|
||||
|
||||
/// Writes one progress event, a JSON object on a line of its own, when
|
||||
/// `--progress json` asked for them. The events, by `event`:
|
||||
///
|
||||
/// - `start`: `accounts`, `parallel`, `dryRun`
|
||||
/// - `setup`: a setup step ended: `step`, `result`, `counts`, `detail`
|
||||
/// - `step`: an account's step began (`result` absent) or ended
|
||||
/// - `account`: an account ended: `email`, `n` of `total`, `result`
|
||||
/// - `end`: `done`, `unfinished`, `report`
|
||||
///
|
||||
/// `result` is `done`, `partial`, `failed`, `done earlier` or `not run`. A
|
||||
/// closed stdout is ignored, as the guide's output is.
|
||||
fn emit(config: &MigrateConfig, event: &str, mut fields: Value) {
|
||||
if !config.progress_json {
|
||||
return;
|
||||
}
|
||||
fields["event"] = json!(event);
|
||||
let _ = writeln!(std::io::stdout().lock(), "{fields}");
|
||||
}
|
||||
|
||||
pub struct Outcome {
|
||||
@@ -76,7 +98,7 @@ struct StepReport {
|
||||
detail: String,
|
||||
}
|
||||
|
||||
#[derive(Default, Clone, Copy)]
|
||||
#[derive(Default, Clone, Copy, Serialize)]
|
||||
struct Counts {
|
||||
fetched: u64,
|
||||
created: u64,
|
||||
@@ -101,6 +123,11 @@ pub fn run(config: MigrateConfig) -> Result<Outcome, Error> {
|
||||
let work = std::path::absolute(&config.work)?;
|
||||
private_dir(&work)?;
|
||||
|
||||
emit(
|
||||
&config,
|
||||
"start",
|
||||
json!({"accounts": accounts.len(), "parallel": config.parallel, "dryRun": config.dry_run}),
|
||||
);
|
||||
let state_path = work.join("state.json");
|
||||
let state = Mutex::new(load_state(&state_path)?);
|
||||
let save = |state: &State| -> Result<(), Error> {
|
||||
@@ -126,7 +153,9 @@ pub fn run(config: MigrateConfig) -> Result<Outcome, Error> {
|
||||
}
|
||||
let log = work.join("setup").join(format!("{}.log", step.name));
|
||||
let ran = run_step(&config, &plan, step, None, &log);
|
||||
setup_reports.push(report_of(&step.name, &ran));
|
||||
let report = report_of(&step.name, &ran);
|
||||
emit(&config, "setup", report.json());
|
||||
setup_reports.push(report);
|
||||
if !config.dry_run {
|
||||
lock(&state).setup.insert(step.name.clone(), ran.result);
|
||||
}
|
||||
@@ -156,6 +185,12 @@ pub fn run(config: MigrateConfig) -> Result<Outcome, Error> {
|
||||
match migrate_account(&config, &plan, &work, email, &state, &save) {
|
||||
Ok(steps) => {
|
||||
let n = finished.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
emit(
|
||||
&config,
|
||||
"account",
|
||||
json!({"email": email, "n": n, "total": total,
|
||||
"result": result_of(&steps)}),
|
||||
);
|
||||
if config.logger.enabled(crate::logging::LEVEL_DEFAULT) {
|
||||
eprintln!("[{n}/{total}] {email}: {}", line_of(&steps));
|
||||
}
|
||||
@@ -187,6 +222,12 @@ pub fn run(config: MigrateConfig) -> Result<Outcome, Error> {
|
||||
})
|
||||
.count();
|
||||
let report = write_report(&work, &config, &setup_reports, &reports)?;
|
||||
emit(
|
||||
&config,
|
||||
"end",
|
||||
json!({"done": reports.len() - unfinished, "unfinished": unfinished,
|
||||
"report": report.display().to_string()}),
|
||||
);
|
||||
Ok(Outcome {
|
||||
done: reports.len() - unfinished,
|
||||
unfinished,
|
||||
@@ -299,8 +340,13 @@ fn migrate_account(
|
||||
}
|
||||
running = true;
|
||||
let log = dir.join(format!("{}.log", step.name));
|
||||
emit(config, "step", json!({"email": email, "step": step.name}));
|
||||
let ran = run_step(config, plan, step, Some((email, &archive)), &log);
|
||||
reports.push(report_of(&step.name, &ran));
|
||||
let report = report_of(&step.name, &ran);
|
||||
let mut ended = report.json();
|
||||
ended["email"] = json!(email);
|
||||
emit(config, "step", ended);
|
||||
reports.push(report);
|
||||
if !config.dry_run {
|
||||
let mut s = lock(state);
|
||||
s.accounts
|
||||
@@ -336,6 +382,24 @@ fn report_of(step: &str, ran: &Ran) -> StepReport {
|
||||
}
|
||||
}
|
||||
|
||||
impl StepReport {
|
||||
fn json(&self) -> Value {
|
||||
json!({
|
||||
"step": self.step, "result": self.result,
|
||||
"counts": self.counts, "detail": self.detail,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// An account's result: the first step that didn't finish decides it.
|
||||
fn result_of(steps: &[StepReport]) -> &'static str {
|
||||
steps
|
||||
.iter()
|
||||
.map(|s| s.result)
|
||||
.find(|r| *r == "failed" || *r == "partial")
|
||||
.unwrap_or("done")
|
||||
}
|
||||
|
||||
fn line_of(steps: &[StepReport]) -> String {
|
||||
match steps
|
||||
.iter()
|
||||
|
||||
@@ -43,6 +43,7 @@ fn config(plan: &Path, work: &Path) -> MigrateConfig {
|
||||
dry_run: false,
|
||||
program: PathBuf::from(env!("CARGO_BIN_EXE_inbuxa-migrate")),
|
||||
logger: Logger::from_flags(true, 0),
|
||||
progress_json: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -178,3 +179,64 @@ fn a_step_sees_only_the_secrets_it_names() {
|
||||
let text = fs::read_to_string(&out.report).unwrap();
|
||||
assert!(text.contains("no password provided"), "{text}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn progress_json_is_one_event_per_line_and_nothing_else() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let root = dir.path();
|
||||
message(&root.join("mail/ann"), "m1", "First");
|
||||
let plan = root.join("plan.json");
|
||||
write_plan(
|
||||
&plan,
|
||||
json!([{"name": "mail", "args": ["import", "maildir", "mail/{local}", "{archive}"]}]),
|
||||
);
|
||||
let out = std::process::Command::new(env!("CARGO_BIN_EXE_inbuxa-migrate"))
|
||||
.args(["migrate", "--parallel", "1", "--progress", "json", "--plan"])
|
||||
.arg(&plan)
|
||||
.arg(root.join("work"))
|
||||
.output()
|
||||
.unwrap();
|
||||
// Bob has no Maildir: the run ends unfinished.
|
||||
assert_eq!(out.status.code(), Some(5));
|
||||
let events: Vec<serde_json::Value> = String::from_utf8(out.stdout)
|
||||
.unwrap()
|
||||
.lines()
|
||||
.map(|l| serde_json::from_str(l).unwrap_or_else(|e| panic!("{e}: {l}")))
|
||||
.collect();
|
||||
let kinds: Vec<&str> = events
|
||||
.iter()
|
||||
.map(|e| e["event"].as_str().unwrap())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
kinds,
|
||||
[
|
||||
"start", "step", "step", "account", "step", "step", "account", "end"
|
||||
]
|
||||
);
|
||||
assert_eq!(events[0]["accounts"], 2);
|
||||
assert_eq!(
|
||||
events[1],
|
||||
json!({"event": "step", "email": "[email protected]", "step": "mail"})
|
||||
);
|
||||
assert_eq!(events[2]["result"], "done");
|
||||
assert_eq!(events[2]["counts"]["created"], 2);
|
||||
assert_eq!(
|
||||
events[3],
|
||||
json!({"event": "account", "email": "[email protected]", "n": 1, "total": 2, "result": "done"})
|
||||
);
|
||||
assert_eq!(events[5]["result"], "failed");
|
||||
assert!(
|
||||
events[5]["detail"].as_str().unwrap().contains("mail/bob"),
|
||||
"{}",
|
||||
events[5]
|
||||
);
|
||||
assert_eq!(events[6]["result"], "failed");
|
||||
assert_eq!(events[7]["done"], 1);
|
||||
assert_eq!(events[7]["unfinished"], 1);
|
||||
assert!(
|
||||
events[7]["report"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.ends_with("report.csv")
|
||||
);
|
||||
}
|
||||
Reference in new issue
Block a user