jcoffey-dev
9faa21f4f1
A live migration on 2026-08-24 stopped a production mail server and then
discovered the host's stalwart-cli was 0.13.4 - present, but from when the
CLI shipped with the server, with no `apply` command. The migration needs
v1.0.2+ from the separately-versioned stalwartlabs/cli repository.
Recovery was closed in both directions. v0.16's recovery-mode boot had
already bumped the store schema to v6, so the 0.15.5 binary refused to
reopen it ("expected 5 or below, found 6"). Going forward needed
export.json, which this tool's own failure path had deleted - and
regenerating it required a settings dump from a live v0.15 instance that
could no longer start. The operator restored a day-old snapshot and lost a
day of mail across nine domains.
Three fixes:
1. preflight.CheckExternalTools verifies stalwart-cli exists and is v1.0.2
or later, and that python3 runs - before anything is touched. Every fact
needed to prevent this was available in under a second from a stopped
state. Skipped for a patch upgrade, which invokes neither tool.
2. A failed run no longer deletes its work directory. Cleaning up on every
exit path was right for a sandboxed rehearsal and catastrophic here:
once the service is stopped the settings dump cannot be regenerated, so
deleting it removes the only way forward. The failure now prints the
resume command instead.
3. `run --resume <id>` continues an interrupted run. The checkpoint
machinery existed but never engaged, because run created a new run every
invocation - so a retry re-ran preflight against a binary already moved
aside, and failed. Completed steps are skipped from the checkpoint.
Proven against a VM built to match the failure: stalwart-cli 0.15.5,
accounts and mail seeded.
* preflight refused, service still active, mail still accepted
* a stub CLI passing --version and failing apply left the run stopped
with all eight inputs intact and the resume command printed
* --resume carried it to a clean finish: five seconds of downtime,
listeners regenerated, admin role restored, quotas rebuilt
That failure-path test is the one that should have run before production.
Every earlier test had stalwart-cli installed from the start, and the one
failure I did exercise happened to leave its artifacts behind.
67 lines
2.7 KiB
Go
67 lines
2.7 KiB
Go
// SPDX-FileCopyrightText: 2026 LINUXexpert-org
|
|
// SPDX-License-Identifier: GPL-3.0-or-later
|
|
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"flag"
|
|
"fmt"
|
|
"os"
|
|
|
|
"github.com/LINUXexpert-org/stalwart-migrator/internal/checkpoint"
|
|
"github.com/LINUXexpert-org/stalwart-migrator/internal/preflight"
|
|
)
|
|
|
|
func runPreflight(args []string) error {
|
|
fs := flag.NewFlagSet("preflight", flag.ExitOnError)
|
|
binaryPath := fs.String("binary", "/usr/local/bin/stalwart", "path to the installed stalwart binary")
|
|
configPath := fs.String("config", "/etc/stalwart/config.toml", "path to stalwart's config file")
|
|
dataDir := fs.String("data-dir", "/var/lib/stalwart", "stalwart data directory")
|
|
containerName := fs.String("container", "stalwart", "docker container name, if applicable")
|
|
adminURL := fs.String("admin-url", "", "base URL for a JMAP reachability check (optional but recommended)")
|
|
adminUser := fs.String("admin-user", "", "admin username for the reachability check")
|
|
adminPassword := fs.String("admin-password", os.Getenv("STALWART_MIGRATE_ADMIN_PASSWORD"),
|
|
"admin password for the reachability check (or set STALWART_MIGRATE_ADMIN_PASSWORD)")
|
|
targetVersion := fs.String("target", "latest", `target Stalwart version, or "latest"`)
|
|
minFree := fs.Float64("min-free-multiple", 2.0, "required free disk space as a multiple of the data directory size")
|
|
stateDir := fs.String("state-dir", checkpoint.DefaultBaseDir, "directory to store run checkpoints in")
|
|
pythonPath := fs.String("python", "python3", "path to python3, needed by migrate_v016.py")
|
|
stalwartCLI := fs.String("stalwart-cli", "stalwart-cli", "path to stalwart-cli (v1.0.2 or later; a separate download from the server)")
|
|
if err := fs.Parse(args); err != nil {
|
|
return err
|
|
}
|
|
|
|
store := checkpoint.NewStore(*stateDir)
|
|
rs, err := store.Create("", *targetVersion)
|
|
if err != nil {
|
|
return fmt.Errorf("create run: %w", err)
|
|
}
|
|
|
|
checker := preflight.New(preflight.Options{
|
|
BinaryPath: *binaryPath,
|
|
ConfigPath: *configPath,
|
|
DataDir: *dataDir,
|
|
ContainerName: *containerName,
|
|
AdminURL: *adminURL,
|
|
AdminUser: *adminUser,
|
|
AdminPassword: *adminPassword,
|
|
TargetVersion: *targetVersion,
|
|
MinFreeMultiple: *minFree,
|
|
CLIPath: *stalwartCLI,
|
|
PythonPath: *pythonPath,
|
|
})
|
|
|
|
report, err := checker.Run(context.Background(), store, rs)
|
|
fmt.Print(report.String())
|
|
fmt.Printf("\nrun id: %s\n", rs.RunID)
|
|
if err != nil {
|
|
return fmt.Errorf("preflight run %s failed to complete: %w", rs.RunID, err)
|
|
}
|
|
if report.Blocking() {
|
|
return fmt.Errorf("preflight found blocking issues for run %s - see FAIL lines above", rs.RunID)
|
|
}
|
|
fmt.Println("preflight passed - safe to proceed with `stalwart-migrate run`")
|
|
return nil
|
|
}
|