Fix the three defects that cost a production restore

A live migration on 2026-08-24 stopped a production mail server and then
discovered the host's stalwart-cli was 0.13.4 - present, but from when the
CLI shipped with the server, with no `apply` command. The migration needs
v1.0.2+ from the separately-versioned stalwartlabs/cli repository.

Recovery was closed in both directions. v0.16's recovery-mode boot had
already bumped the store schema to v6, so the 0.15.5 binary refused to
reopen it ("expected 5 or below, found 6"). Going forward needed
export.json, which this tool's own failure path had deleted - and
regenerating it required a settings dump from a live v0.15 instance that
could no longer start. The operator restored a day-old snapshot and lost a
day of mail across nine domains.

Three fixes:

1. preflight.CheckExternalTools verifies stalwart-cli exists and is v1.0.2
   or later, and that python3 runs - before anything is touched. Every fact
   needed to prevent this was available in under a second from a stopped
   state. Skipped for a patch upgrade, which invokes neither tool.

2. A failed run no longer deletes its work directory. Cleaning up on every
   exit path was right for a sandboxed rehearsal and catastrophic here:
   once the service is stopped the settings dump cannot be regenerated, so
   deleting it removes the only way forward. The failure now prints the
   resume command instead.

3. `run --resume <id>` continues an interrupted run. The checkpoint
   machinery existed but never engaged, because run created a new run every
   invocation - so a retry re-ran preflight against a binary already moved
   aside, and failed. Completed steps are skipped from the checkpoint.

Proven against a VM built to match the failure: stalwart-cli 0.15.5,
accounts and mail seeded.

  * preflight refused, service still active, mail still accepted
  * a stub CLI passing --version and failing apply left the run stopped
    with all eight inputs intact and the resume command printed
  * --resume carried it to a clean finish: five seconds of downtime,
    listeners regenerated, admin role restored, quotas rebuilt

That failure-path test is the one that should have run before production.
Every earlier test had stalwart-cli installed from the start, and the one
failure I did exercise happened to leave its artifacts behind.

cmd/stalwart-migrate/preflight.go

@@ -26,6 +26,8 @@ func runPreflight(args []string) error {
 	targetVersion := fs.String("target", "latest", `target Stalwart version, or "latest"`)
 	minFree := fs.Float64("min-free-multiple", 2.0, "required free disk space as a multiple of the data directory size")
 	stateDir := fs.String("state-dir", checkpoint.DefaultBaseDir, "directory to store run checkpoints in")
+	pythonPath := fs.String("python", "python3", "path to python3, needed by migrate_v016.py")
+	stalwartCLI := fs.String("stalwart-cli", "stalwart-cli", "path to stalwart-cli (v1.0.2 or later; a separate download from the server)")
 	if err := fs.Parse(args); err != nil {
 		return err
 	}
@@ -46,6 +48,8 @@ func runPreflight(args []string) error {
 		AdminPassword:   *adminPassword,
 		TargetVersion:   *targetVersion,
 		MinFreeMultiple: *minFree,
+		CLIPath:         *stalwartCLI,
+		PythonPath:      *pythonPath,
 	})
 
 	report, err := checker.Run(context.Background(), store, rs)

cmd/stalwart-migrate/rehearse.go

@@ -52,6 +52,8 @@ func runRehearse(args []string) (err error) {
 	stateDir := fs.String("state-dir", checkpoint.DefaultBaseDir, "directory to store run checkpoints in")
 	workDir := fs.String("work-dir", "/var/lib/stalwart-migrator/work", "scratch directory for the dumps and converted plan (cleaned up afterward - see --keep-artifacts)")
 	pythonPath := fs.String("python", "python3", "path to python3")
+	stalwartCLI := fs.String("stalwart-cli", "stalwart-cli",
+		"path to stalwart-cli (v1.0.2 or later; a separate download from the server) - rehearsal doesn't invoke it, but checks it so `run` doesn't fail after stopping the service")
 	migrationScriptSHA256 := fs.String("migration-script-sha256", "", "pinned sha256 of migrate_v016.py (recommended; the first run prints the hash to pin)")
 	minFree := fs.Float64("min-free-multiple", 2.0, "free-space multiple preflight checks for; rehearsal itself copies nothing")
 	keepArtifacts := fs.Bool("keep-artifacts", false, "don't delete work-dir/<run-id> afterward")
@@ -94,6 +96,10 @@ func runRehearse(args []string) (err error) {
 			fmt.Printf("\nartifacts kept at %s (--keep-artifacts)\n", runWorkDir)
 			return
 		}
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "\nthe rehearsal failed; its artifacts are kept at %s for inspection\n", runWorkDir)
+			return
+		}
 		if rmErr := os.RemoveAll(runWorkDir); rmErr != nil {
 			fmt.Fprintf(os.Stderr, "\nwarning: failed to clean up %s: %v (remove it manually)\n", runWorkDir, rmErr)
 			return
@@ -106,6 +112,7 @@ func runRehearse(args []string) (err error) {
 		BinaryPath: *binaryPath, ConfigPath: *configPath, DataDir: *dataDir, ContainerName: *containerName,
 		AdminURL: *adminURL, AdminUser: *adminUser, AdminPassword: *adminPassword,
 		TargetVersion: *targetVersion, MinFreeMultiple: *minFree, HTTPClient: httpClient,
+		CLIPath: *stalwartCLI, PythonPath: *pythonPath,
 	})
 	pfReport, err := checker.Run(ctx, store, rs)
 	fmt.Print(pfReport.String())

cmd/stalwart-migrate/run.go

@@ -69,6 +69,7 @@ func runRun(args []string) (err error) {
 	minFree := fs.Float64("min-free-multiple", 2.0, "required free disk space as a multiple of the data directory size")
 	recalcQuotas := fs.Bool("recalculate-quotas", true, "schedule the post-migration quota rebuild")
 	keepArtifacts := fs.Bool("keep-artifacts", false, "don't delete work-dir/<run-id> afterward")
+	resume := fs.String("resume", "", "resume an interrupted run by id instead of starting a new one (see `status` for ids)")
 	yes := fs.Bool("yes", false, "actually perform the migration")
 	recoveryConfirmed := fs.Bool("recovery-point-confirmed", false,
 		"confirm you have a snapshot or backup you have verified you can restore - this tool cannot undo a migration")
@@ -104,11 +105,25 @@ func runRun(args []string) (err error) {
 	}
 
 	store := checkpoint.NewStore(*stateDir)
-	rs, err := store.Create("", *targetVersion)
-	if err != nil {
-		return fmt.Errorf("create run: %w", err)
+	var rs *checkpoint.RunState
+	if *resume != "" {
+		// Resuming is not a convenience. A run that fails partway leaves
+		// the service stopped and the store part-migrated, and starting
+		// over is often impossible: preflight would re-run against a
+		// binary that has already been moved aside, and the settings dump
+		// needs a live pre-migration instance that no longer exists.
+		// Completed steps are skipped from the checkpoint, so this picks
+		// up where it stopped.
+		if rs, err = store.Load(*resume); err != nil {
+			return fmt.Errorf("resume run %s: %w", *resume, err)
+		}
+		fmt.Printf("\nresuming run: %s (completed steps will be skipped)\n", rs.RunID)
+	} else {
+		if rs, err = store.Create("", *targetVersion); err != nil {
+			return fmt.Errorf("create run: %w", err)
+		}
+		fmt.Printf("\nrun id: %s\n", rs.RunID)
 	}
-	fmt.Printf("\nrun id: %s\n", rs.RunID)
 	runWorkDir := filepath.Join(*workDir, rs.RunID)
 	runStateDir := filepath.Join(*stateDir, rs.RunID)
 	if err := os.MkdirAll(runWorkDir, 0o750); err != nil {
@@ -119,6 +134,18 @@ func runRun(args []string) (err error) {
 			fmt.Printf("\nartifacts kept at %s (--keep-artifacts)\n", runWorkDir)
 			return
 		}
+		if err != nil {
+			// Never clean up after a failure. These files - the settings
+			// dump, the converted config and export plan - are the run's
+			// inputs, and after the service has been stopped they cannot
+			// be regenerated: the dump needs a live pre-migration instance.
+			// Deleting them once turned a missing-dependency error into a
+			// restore-from-snapshot, because there was no way forward and
+			// no way back.
+			fmt.Fprintf(os.Stderr, "\nthe run failed; its artifacts are kept at %s\n", runWorkDir)
+			fmt.Fprintf(os.Stderr, "resume it once the cause is fixed:\n  stalwart-migrate run --resume %s [same flags]\n", rs.RunID)
+			return
+		}
 		if rmErr := os.RemoveAll(runWorkDir); rmErr != nil {
 			fmt.Fprintf(os.Stderr, "warning: couldn't clean up %s: %v\n", runWorkDir, rmErr)
 		}
@@ -129,6 +156,7 @@ func runRun(args []string) (err error) {
 		BinaryPath: *binaryPath, ConfigPath: *configPath, DataDir: *dataDir, ContainerName: *containerName,
 		AdminURL: *adminURL, AdminUser: *adminUser, AdminPassword: *adminPassword,
 		TargetVersion: *targetVersion, MinFreeMultiple: *minFree, HTTPClient: httpClient,
+		CLIPath: *stalwartCLI, PythonPath: *pythonPath,
 	}).Run(ctx, store, rs)
 	fmt.Print(pfReport.String())
 	if err != nil {