jcoffey-dev
9faa21f4f1
A live migration on 2026-08-24 stopped a production mail server and then
discovered the host's stalwart-cli was 0.13.4 - present, but from when the
CLI shipped with the server, with no `apply` command. The migration needs
v1.0.2+ from the separately-versioned stalwartlabs/cli repository.
Recovery was closed in both directions. v0.16's recovery-mode boot had
already bumped the store schema to v6, so the 0.15.5 binary refused to
reopen it ("expected 5 or below, found 6"). Going forward needed
export.json, which this tool's own failure path had deleted - and
regenerating it required a settings dump from a live v0.15 instance that
could no longer start. The operator restored a day-old snapshot and lost a
day of mail across nine domains.
Three fixes:
1. preflight.CheckExternalTools verifies stalwart-cli exists and is v1.0.2
or later, and that python3 runs - before anything is touched. Every fact
needed to prevent this was available in under a second from a stopped
state. Skipped for a patch upgrade, which invokes neither tool.
2. A failed run no longer deletes its work directory. Cleaning up on every
exit path was right for a sandboxed rehearsal and catastrophic here:
once the service is stopped the settings dump cannot be regenerated, so
deleting it removes the only way forward. The failure now prints the
resume command instead.
3. `run --resume <id>` continues an interrupted run. The checkpoint
machinery existed but never engaged, because run created a new run every
invocation - so a retry re-ran preflight against a binary already moved
aside, and failed. Completed steps are skipped from the checkpoint.
Proven against a VM built to match the failure: stalwart-cli 0.15.5,
accounts and mail seeded.
* preflight refused, service still active, mail still accepted
* a stub CLI passing --version and failing apply left the run stopped
with all eight inputs intact and the resume command printed
* --resume carried it to a clean finish: five seconds of downtime,
listeners regenerated, admin role restored, quotas rebuilt
That failure-path test is the one that should have run before production.
Every earlier test had stalwart-cli installed from the start, and the one
failure I did exercise happened to leave its artifacts behind.
103 lines
3.5 KiB
Go
103 lines
3.5 KiB
Go
// SPDX-FileCopyrightText: 2026 LINUXexpert-org
|
|
// SPDX-License-Identifier: GPL-3.0-or-later
|
|
|
|
package preflight
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// fakeTool puts an executable of the given name at the front of PATH,
|
|
// reporting the given --version output.
|
|
func fakeTool(t *testing.T, name, versionOutput string) {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
script := "#!/bin/sh\necho '" + versionOutput + "'\n"
|
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(script), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
|
|
}
|
|
|
|
func statusOf(results []CheckResult, name string) (Status, string) {
|
|
for _, r := range results {
|
|
if r.Name == name {
|
|
return r.Status, r.Detail
|
|
}
|
|
}
|
|
return "", ""
|
|
}
|
|
|
|
// This is the check whose absence took a production mail server down: a
|
|
// live migration stopped the service, then discovered the host's
|
|
// stalwart-cli was 0.13.4 and had no `apply` command. Recovery cost a
|
|
// restore from a day-old snapshot.
|
|
func TestCheckExternalToolsFailsWhenTheCLIIsMissing(t *testing.T) {
|
|
t.Setenv("PATH", t.TempDir()) // nothing on PATH at all
|
|
results := CheckExternalTools(context.Background(), "", "", true)
|
|
|
|
status, detail := statusOf(results, "stalwart-cli")
|
|
if status != StatusFail {
|
|
t.Errorf("stalwart-cli = %q, want fail - this must stop the run before the service does", status)
|
|
}
|
|
if !strings.Contains(detail, "separate download") {
|
|
t.Errorf("detail %q should say it's a separate download from the server", detail)
|
|
}
|
|
if !strings.Contains(detail, "v1.0.2") {
|
|
t.Errorf("detail %q should name the minimum version", detail)
|
|
}
|
|
}
|
|
|
|
// The exact version that was on the production host. It exists, it runs,
|
|
// and it cannot do the job.
|
|
func TestCheckExternalToolsFailsOnTheOldBundledCLI(t *testing.T) {
|
|
fakeTool(t, "stalwart-cli", "stalwart-cli 0.13.4")
|
|
fakeTool(t, "python3", "Python 3.13.5")
|
|
results := CheckExternalTools(context.Background(), "", "", true)
|
|
|
|
status, detail := statusOf(results, "stalwart-cli")
|
|
if status != StatusFail {
|
|
t.Errorf("stalwart-cli 0.13.4 = %q, want fail", status)
|
|
}
|
|
if !strings.Contains(detail, "0.13.4") || !strings.Contains(detail, "no `apply` command") {
|
|
t.Errorf("detail %q should name the version found and why it won't do", detail)
|
|
}
|
|
}
|
|
|
|
func TestCheckExternalToolsAcceptsASupportedCLI(t *testing.T) {
|
|
fakeTool(t, "stalwart-cli", "stalwart-cli 1.0.12")
|
|
fakeTool(t, "python3", "Python 3.13.5")
|
|
results := CheckExternalTools(context.Background(), "", "", true)
|
|
|
|
for _, name := range []string{"stalwart-cli", "python3"} {
|
|
if status, detail := statusOf(results, name); status != StatusOK {
|
|
t.Errorf("%s = %q (%s), want ok", name, status, detail)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestCheckExternalToolsFailsWithoutPython(t *testing.T) {
|
|
fakeTool(t, "stalwart-cli", "stalwart-cli 1.0.12")
|
|
results := CheckExternalTools(context.Background(), "", "/nonexistent/python3", true)
|
|
if status, _ := statusOf(results, "python3"); status != StatusFail {
|
|
t.Errorf("python3 = %q, want fail - migrate_v016.py cannot run without it", status)
|
|
}
|
|
}
|
|
|
|
// A patch bump replays no settings, so neither tool is invoked and a
|
|
// missing CLI must not block it.
|
|
func TestCheckExternalToolsSkipsForAPatchUpgrade(t *testing.T) {
|
|
t.Setenv("PATH", t.TempDir())
|
|
results := CheckExternalTools(context.Background(), "", "", false)
|
|
if len(results) != 1 {
|
|
t.Fatalf("got %d result(s), want a single skip-style result", len(results))
|
|
}
|
|
if results[0].Status != StatusOK {
|
|
t.Errorf("status = %q, want ok for a patch upgrade with no tools present", results[0].Status)
|
|
}
|
|
}
|