Files
inbuxa-migrate/tests/mock_migrate.rs
T
jcoffey-dev 4a14d0f6b6
ci / test (pull_request) Skipped
github/ci (branch) GitHub Actions
ci / github (pull_request) Successful in 7m12s
ci / announce (pull_request) Skipped
migrate --progress json: one event per line, for a program running it
The migrations page in inbuxa Admin runs migrate as a child of the server
and shows its progress. Text lines were made for people; these events are
for that: start, setup, step begun or ended (with result, counts and
detail), account ended, and end with the report's path. Nothing else goes
to stdout in that mode.
2026-10-30 17:41:06 -07:00

243 lines
8.1 KiB
Rust

/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC <[email protected]>
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*/
//! `migrate` runs a plan's steps for each account as runs of the real
//! binary: Maildir in, files out, offline. One account fails and the rest
//! go on; a second run resumes only what didn't finish; `--again` runs it
//! all once more. Counts in the report are summed over the types: a
//! mailbox and an email are two.
use std::fs;
use std::path::{Path, PathBuf};
use inbuxa_migrate::logging::Logger;
use inbuxa_migrate::migrate::{self, MigrateConfig, Outcome};
use rusqlite::Connection;
use serde_json::json;
use tempfile::TempDir;
fn message(dir: &Path, name: &str, subject: &str) {
for sub in ["cur", "new", "tmp"] {
fs::create_dir_all(dir.join(sub)).unwrap();
}
fs::write(
dir.join("cur").join(format!("{name}:2,S")),
format!(
"From: [email protected]\r\nTo: [email protected]\r\nSubject: {subject}\r\n\
Message-ID: <{name}@example.org>\r\nDate: Mon, 4 Mar 2024 09:00:00 +0000\r\n\r\nHi\r\n"
),
)
.unwrap();
}
fn config(plan: &Path, work: &Path) -> MigrateConfig {
MigrateConfig {
plan: plan.to_path_buf(),
work: work.to_path_buf(),
parallel: 2,
again: false,
only: Vec::new(),
dry_run: false,
program: PathBuf::from(env!("CARGO_BIN_EXE_inbuxa-migrate")),
logger: Logger::from_flags(true, 0),
progress_json: false,
}
}
fn emails(archive: &Path) -> i64 {
Connection::open(archive)
.unwrap()
.query_row("SELECT COUNT(*) FROM emails", [], |r| r.get(0))
.unwrap()
}
fn report(outcome: &Outcome) -> Vec<String> {
fs::read_to_string(&outcome.report)
.unwrap()
.lines()
.skip(1)
.map(|l| l.split(',').take(5).collect::<Vec<_>>().join(","))
.collect()
}
fn write_plan(path: &Path, steps: serde_json::Value) {
let plan = json!({
"format": "inbuxa-migrate-plan",
"formatVersion": 1,
"accounts": ["[email protected]", "[email protected]"],
"steps": steps,
});
fs::write(path, serde_json::to_vec_pretty(&plan).unwrap()).unwrap();
}
#[test]
fn accounts_run_their_steps_resume_and_run_again() {
let dir = TempDir::new().unwrap();
let root = dir.path();
message(&root.join("mail/ann"), "m1", "First");
let plan = root.join("plan.json");
// Relative paths are the plan's own: the steps run in its directory.
write_plan(
&plan,
json!([
{"name": "mail", "args": ["import", "maildir", "mail/{local}", "{archive}"]},
{"name": "files", "args": ["export-files", "{archive}", "out/{email}"]},
]),
);
let work = root.join("work");
// Bob has no Maildir yet: his import fails, his export never runs.
let first = migrate::run(config(&plan, &work)).unwrap();
assert_eq!((first.done, first.unfinished), (1, 1));
assert_eq!(
report(&first),
[
"[email protected],mail,done,2,2",
"[email protected],files,done,0,2",
"[email protected],mail,failed,0,0",
"[email protected],files,not run,0,0",
]
);
let detail = fs::read_to_string(&first.report).unwrap();
assert!(
detail.contains("mail/bob"),
"the reason is reported: {detail}"
);
assert_eq!(emails(&work.join("[email protected]/archive.sqlite")), 1);
assert!(root.join("out/[email protected]/manifest.json").exists());
let log = fs::read_to_string(work.join("[email protected]/mail.log")).unwrap();
assert!(
log.starts_with("== inbuxa-migrate import maildir mail/ann "),
"{log}"
);
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mode = |p: &Path| fs::metadata(p).unwrap().permissions().mode() & 0o777;
assert_eq!(mode(&work), 0o700);
assert_eq!(mode(&work.join("[email protected]/mail.log")), 0o600);
}
// The second run leaves Ann alone (her export-files would refuse a
// directory that isn't empty) and finishes Bob.
message(&root.join("mail/bob"), "m2", "Second");
let second = migrate::run(config(&plan, &work)).unwrap();
assert_eq!((second.done, second.unfinished), (2, 0));
assert_eq!(
report(&second),
[
"[email protected],mail,done earlier,0,0",
"[email protected],files,done earlier,0,0",
"[email protected],mail,done,2,2",
"[email protected],files,done,0,2",
]
);
// --again: every step runs once more and picks up what's new.
write_plan(
&plan,
json!([{"name": "mail", "args": ["import", "maildir", "mail/{local}", "{archive}"]}]),
);
message(&root.join("mail/ann"), "m3", "Third");
let mut again = config(&plan, &work);
again.again = true;
again.only = vec!["[email protected]".into()];
let third = migrate::run(again).unwrap();
assert_eq!(report(&third), ["[email protected],mail,done,1,1"]);
assert_eq!(emails(&work.join("[email protected]/archive.sqlite")), 2);
}
#[test]
fn a_step_sees_only_the_secrets_it_names() {
let dir = TempDir::new().unwrap();
let plan = dir.path().join("plan.json");
let step = |secrets: serde_json::Value| {
json!([{
"name": "mail",
"args": ["import", "imap", "--url", "imap://127.0.0.1:9", "--allow-cleartext",
"--auth-basic", "{email}", "{archive}"],
"secrets": secrets,
}])
};
write_plan(&plan, step(json!({"password": "MIGRATE_TEST_UNSET_VAR"})));
let err = migrate::run(config(&plan, &dir.path().join("w1")))
.err()
.unwrap()
.to_string();
assert!(err.contains("$MIGRATE_TEST_UNSET_VAR is not set"), "{err}");
// No secret named: the password this test run may have in its own
// environment is not passed on, and with no terminal the step fails
// at once rather than waiting for one.
write_plan(&plan, step(json!({})));
let out = migrate::run(config(&plan, &dir.path().join("w2"))).unwrap();
assert_eq!(out.unfinished, 2);
let text = fs::read_to_string(&out.report).unwrap();
assert!(text.contains("no password provided"), "{text}");
}
#[test]
fn progress_json_is_one_event_per_line_and_nothing_else() {
let dir = TempDir::new().unwrap();
let root = dir.path();
message(&root.join("mail/ann"), "m1", "First");
let plan = root.join("plan.json");
write_plan(
&plan,
json!([{"name": "mail", "args": ["import", "maildir", "mail/{local}", "{archive}"]}]),
);
let out = std::process::Command::new(env!("CARGO_BIN_EXE_inbuxa-migrate"))
.args(["migrate", "--parallel", "1", "--progress", "json", "--plan"])
.arg(&plan)
.arg(root.join("work"))
.output()
.unwrap();
// Bob has no Maildir: the run ends unfinished.
assert_eq!(out.status.code(), Some(5));
let events: Vec<serde_json::Value> = String::from_utf8(out.stdout)
.unwrap()
.lines()
.map(|l| serde_json::from_str(l).unwrap_or_else(|e| panic!("{e}: {l}")))
.collect();
let kinds: Vec<&str> = events
.iter()
.map(|e| e["event"].as_str().unwrap())
.collect();
assert_eq!(
kinds,
[
"start", "step", "step", "account", "step", "step", "account", "end"
]
);
assert_eq!(events[0]["accounts"], 2);
assert_eq!(
events[1],
json!({"event": "step", "email": "[email protected]", "step": "mail"})
);
assert_eq!(events[2]["result"], "done");
assert_eq!(events[2]["counts"]["created"], 2);
assert_eq!(
events[3],
json!({"event": "account", "email": "[email protected]", "n": 1, "total": 2, "result": "done"})
);
assert_eq!(events[5]["result"], "failed");
assert!(
events[5]["detail"].as_str().unwrap().contains("mail/bob"),
"{}",
events[5]
);
assert_eq!(events[6]["result"], "failed");
assert_eq!(events[7]["done"], 1);
assert_eq!(events[7]["unfinished"], 1);
assert!(
events[7]["report"]
.as_str()
.unwrap()
.ends_with("report.csv")
);
}