Files
inbuxa-migrate/tests/mock_migrate.rs
T
jcoffey-dev 4bbce61615
ci / test (pull_request) Skipped
github/ci (branch) GitHub Actions
ci / github (pull_request) Successful in 6m12s
ci / announce (pull_request) Skipped
migrate: --repeat, keep syncing until the cutover
Moving a busy account means moving it twice: the bulk ahead of time, then
a short catch-up at the cutover so almost no mail is lost to downtime. The
import and export steps already converge -- each run moves only what
changed -- so the catch-up was there, but driving it was manual: run,
wait, run again.

--repeat <DURATION> drives it. After each pass over the accounts it waits
that long and runs them again, each pass moving only what has arrived at
the source since the last, until the run is interrupted. The setup runs
once, before the first pass. Leave it running while the old server still
takes mail; change the MX and stop it, and the final pass is a short
catch-up, not the whole mailbox. A pass in flight is resumable, so Ctrl-C
is safe.

--max-passes <N> bounds it, and --until-quiet stops on the first pass that
moves nothing new -- the source has gone quiet and the servers are in
step, a cue the cutover is safe. Each pass re-runs every step, like
--again, which is what makes it catch the delta rather than skip the
done steps.

The account pass is factored into run_accounts_pass and the run loops it;
Outcome gains the pass count, the json progress gains a per-pass end event
and a waiting event between passes. Tests cover the multi-pass loop, the
--until-quiet stop and the duration parser.
2026-11-12 11:17:10 -07:00

304 lines
10 KiB
Rust

/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC <[email protected]>
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*/
//! `migrate` runs a plan's steps for each account as runs of the real
//! binary: Maildir in, files out, offline. One account fails and the rest
//! go on; a second run resumes only what didn't finish; `--again` runs it
//! all once more. Counts in the report are summed over the types: a
//! mailbox and an email are two.
use std::fs;
use std::path::{Path, PathBuf};
use inbuxa_migrate::logging::Logger;
use inbuxa_migrate::migrate::{self, MigrateConfig, Outcome};
use rusqlite::Connection;
use serde_json::json;
use tempfile::TempDir;
fn message(dir: &Path, name: &str, subject: &str) {
for sub in ["cur", "new", "tmp"] {
fs::create_dir_all(dir.join(sub)).unwrap();
}
fs::write(
dir.join("cur").join(format!("{name}:2,S")),
format!(
"From: [email protected]\r\nTo: [email protected]\r\nSubject: {subject}\r\n\
Message-ID: <{name}@example.org>\r\nDate: Mon, 4 Mar 2024 09:00:00 +0000\r\n\r\nHi\r\n"
),
)
.unwrap();
}
fn config(plan: &Path, work: &Path) -> MigrateConfig {
MigrateConfig {
plan: plan.to_path_buf(),
work: work.to_path_buf(),
parallel: 2,
again: false,
only: Vec::new(),
dry_run: false,
program: PathBuf::from(env!("CARGO_BIN_EXE_inbuxa-migrate")),
logger: Logger::from_flags(true, 0),
progress_json: false,
repeat: None,
max_passes: None,
until_quiet: false,
}
}
fn emails(archive: &Path) -> i64 {
Connection::open(archive)
.unwrap()
.query_row("SELECT COUNT(*) FROM emails", [], |r| r.get(0))
.unwrap()
}
fn report(outcome: &Outcome) -> Vec<String> {
fs::read_to_string(&outcome.report)
.unwrap()
.lines()
.skip(1)
.map(|l| l.split(',').take(5).collect::<Vec<_>>().join(","))
.collect()
}
fn write_plan(path: &Path, steps: serde_json::Value) {
let plan = json!({
"format": "inbuxa-migrate-plan",
"formatVersion": 1,
"accounts": ["[email protected]", "[email protected]"],
"steps": steps,
});
fs::write(path, serde_json::to_vec_pretty(&plan).unwrap()).unwrap();
}
#[test]
fn accounts_run_their_steps_resume_and_run_again() {
let dir = TempDir::new().unwrap();
let root = dir.path();
message(&root.join("mail/ann"), "m1", "First");
let plan = root.join("plan.json");
// Relative paths are the plan's own: the steps run in its directory.
write_plan(
&plan,
json!([
{"name": "mail", "args": ["import", "maildir", "mail/{local}", "{archive}"]},
{"name": "files", "args": ["export-files", "{archive}", "out/{email}"]},
]),
);
let work = root.join("work");
// Bob has no Maildir yet: his import fails, his export never runs.
let first = migrate::run(config(&plan, &work)).unwrap();
assert_eq!((first.done, first.unfinished), (1, 1));
assert_eq!(
report(&first),
[
"[email protected],mail,done,2,2",
"[email protected],files,done,0,2",
"[email protected],mail,failed,0,0",
"[email protected],files,not run,0,0",
]
);
let detail = fs::read_to_string(&first.report).unwrap();
assert!(
detail.contains("mail/bob"),
"the reason is reported: {detail}"
);
assert_eq!(emails(&work.join("[email protected]/archive.sqlite")), 1);
assert!(root.join("out/[email protected]/manifest.json").exists());
let log = fs::read_to_string(work.join("[email protected]/mail.log")).unwrap();
assert!(
log.starts_with("== inbuxa-migrate import maildir mail/ann "),
"{log}"
);
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mode = |p: &Path| fs::metadata(p).unwrap().permissions().mode() & 0o777;
assert_eq!(mode(&work), 0o700);
assert_eq!(mode(&work.join("[email protected]/mail.log")), 0o600);
}
// The second run leaves Ann alone (her export-files would refuse a
// directory that isn't empty) and finishes Bob.
message(&root.join("mail/bob"), "m2", "Second");
let second = migrate::run(config(&plan, &work)).unwrap();
assert_eq!((second.done, second.unfinished), (2, 0));
assert_eq!(
report(&second),
[
"[email protected],mail,done earlier,0,0",
"[email protected],files,done earlier,0,0",
"[email protected],mail,done,2,2",
"[email protected],files,done,0,2",
]
);
// --again: every step runs once more and picks up what's new.
write_plan(
&plan,
json!([{"name": "mail", "args": ["import", "maildir", "mail/{local}", "{archive}"]}]),
);
message(&root.join("mail/ann"), "m3", "Third");
let mut again = config(&plan, &work);
again.again = true;
again.only = vec!["[email protected]".into()];
let third = migrate::run(again).unwrap();
assert_eq!(report(&third), ["[email protected],mail,done,1,1"]);
assert_eq!(emails(&work.join("[email protected]/archive.sqlite")), 2);
}
#[test]
fn repeat_runs_several_passes_and_stops_when_the_source_is_quiet() {
let dir = TempDir::new().unwrap();
let root = dir.path();
message(&root.join("mail/ann"), "m1", "First");
let plan = root.join("plan.json");
write_plan(
&plan,
json!([{"name": "mail", "args": ["import", "maildir", "mail/{local}", "{archive}"]}]),
);
let work = root.join("work");
// --repeat with a bound: every pass re-runs the import. The first moves
// the one message; later passes find nothing new, so with --until-quiet
// the run stops on the first quiet pass -- here the second -- without
// reaching the three-pass ceiling.
let mut cfg = config(&plan, &work);
cfg.only = vec!["[email protected]".into()];
cfg.repeat = Some(std::time::Duration::from_millis(5));
cfg.max_passes = Some(3);
cfg.until_quiet = true;
let out = migrate::run(cfg).unwrap();
assert_eq!(
out.passes, 2,
"pass 1 moves mail, pass 2 is quiet and stops"
);
assert_eq!((out.done, out.unfinished), (1, 0));
// The message landed once, not once per pass.
assert_eq!(emails(&work.join("[email protected]/archive.sqlite")), 1);
// The final report is the quiet pass: the re-run found nothing new.
assert_eq!(report(&out), ["[email protected],mail,done,0,0"]);
}
#[test]
fn repeat_with_a_ceiling_runs_exactly_that_many_passes() {
let dir = TempDir::new().unwrap();
let root = dir.path();
message(&root.join("mail/ann"), "m1", "First");
let plan = root.join("plan.json");
write_plan(
&plan,
json!([{"name": "mail", "args": ["import", "maildir", "mail/{local}", "{archive}"]}]),
);
let work = root.join("work");
// Without --until-quiet the run keeps going to the ceiling even though
// later passes are quiet, and the message is still imported only once.
let mut cfg = config(&plan, &work);
cfg.only = vec!["[email protected]".into()];
cfg.repeat = Some(std::time::Duration::from_millis(5));
cfg.max_passes = Some(3);
let out = migrate::run(cfg).unwrap();
assert_eq!(out.passes, 3);
assert_eq!(emails(&work.join("[email protected]/archive.sqlite")), 1);
}
#[test]
fn a_step_sees_only_the_secrets_it_names() {
let dir = TempDir::new().unwrap();
let plan = dir.path().join("plan.json");
let step = |secrets: serde_json::Value| {
json!([{
"name": "mail",
"args": ["import", "imap", "--url", "imap://127.0.0.1:9", "--allow-cleartext",
"--auth-basic", "{email}", "{archive}"],
"secrets": secrets,
}])
};
write_plan(&plan, step(json!({"password": "MIGRATE_TEST_UNSET_VAR"})));
let err = migrate::run(config(&plan, &dir.path().join("w1")))
.err()
.unwrap()
.to_string();
assert!(err.contains("$MIGRATE_TEST_UNSET_VAR is not set"), "{err}");
// No secret named: the password this test run may have in its own
// environment is not passed on, and with no terminal the step fails
// at once rather than waiting for one.
write_plan(&plan, step(json!({})));
let out = migrate::run(config(&plan, &dir.path().join("w2"))).unwrap();
assert_eq!(out.unfinished, 2);
let text = fs::read_to_string(&out.report).unwrap();
assert!(text.contains("no password provided"), "{text}");
}
#[test]
fn progress_json_is_one_event_per_line_and_nothing_else() {
let dir = TempDir::new().unwrap();
let root = dir.path();
message(&root.join("mail/ann"), "m1", "First");
let plan = root.join("plan.json");
write_plan(
&plan,
json!([{"name": "mail", "args": ["import", "maildir", "mail/{local}", "{archive}"]}]),
);
let out = std::process::Command::new(env!("CARGO_BIN_EXE_inbuxa-migrate"))
.args(["migrate", "--parallel", "1", "--progress", "json", "--plan"])
.arg(&plan)
.arg(root.join("work"))
.output()
.unwrap();
// Bob has no Maildir: the run ends unfinished.
assert_eq!(out.status.code(), Some(5));
let events: Vec<serde_json::Value> = String::from_utf8(out.stdout)
.unwrap()
.lines()
.map(|l| serde_json::from_str(l).unwrap_or_else(|e| panic!("{e}: {l}")))
.collect();
let kinds: Vec<&str> = events
.iter()
.map(|e| e["event"].as_str().unwrap())
.collect();
assert_eq!(
kinds,
[
"start", "step", "step", "account", "step", "step", "account", "end"
]
);
assert_eq!(events[0]["accounts"], 2);
assert_eq!(
events[1],
json!({"event": "step", "email": "[email protected]", "step": "mail"})
);
assert_eq!(events[2]["result"], "done");
assert_eq!(events[2]["counts"]["created"], 2);
assert_eq!(
events[3],
json!({"event": "account", "email": "[email protected]", "n": 1, "total": 2, "result": "done"})
);
assert_eq!(events[5]["result"], "failed");
assert!(
events[5]["detail"].as_str().unwrap().contains("mail/bob"),
"{}",
events[5]
);
assert_eq!(events[6]["result"], "failed");
assert_eq!(events[7]["done"], 1);
assert_eq!(events[7]["unfinished"], 1);
assert!(
events[7]["report"]
.as_str()
.unwrap()
.ends_with("report.csv")
);
}