A live migration on 2026-08-24 stopped a production mail server and then
discovered the host's stalwart-cli was 0.13.4 - present, but from when the
CLI shipped with the server, with no `apply` command. The migration needs
v1.0.2+ from the separately-versioned stalwartlabs/cli repository.
Recovery was closed in both directions. v0.16's recovery-mode boot had
already bumped the store schema to v6, so the 0.15.5 binary refused to
reopen it ("expected 5 or below, found 6"). Going forward needed
export.json, which this tool's own failure path had deleted - and
regenerating it required a settings dump from a live v0.15 instance that
could no longer start. The operator restored a day-old snapshot and lost a
day of mail across nine domains.
Three fixes:
1. preflight.CheckExternalTools verifies stalwart-cli exists and is v1.0.2
or later, and that python3 runs - before anything is touched. Every fact
needed to prevent this was available in under a second from a stopped
state. Skipped for a patch upgrade, which invokes neither tool.
2. A failed run no longer deletes its work directory. Cleaning up on every
exit path was right for a sandboxed rehearsal and catastrophic here:
once the service is stopped the settings dump cannot be regenerated, so
deleting it removes the only way forward. The failure now prints the
resume command instead.
3. `run --resume <id>` continues an interrupted run. The checkpoint
machinery existed but never engaged, because run created a new run every
invocation - so a retry re-ran preflight against a binary already moved
aside, and failed. Completed steps are skipped from the checkpoint.
Proven against a VM built to match the failure: stalwart-cli 0.15.5,
accounts and mail seeded.
* preflight refused, service still active, mail still accepted
* a stub CLI passing --version and failing apply left the run stopped
with all eight inputs intact and the resume command printed
* --resume carried it to a clean finish: five seconds of downtime,
listeners regenerated, admin role restored, quotas rebuilt
That failure-path test is the one that should have run before production.
Every earlier test had stalwart-cli installed from the start, and the one
failure I did exercise happened to leave its artifacts behind.
168 lines
5.9 KiB
Go
168 lines
5.9 KiB
Go
// SPDX-FileCopyrightText: 2026 LINUXexpert-org
|
|
// SPDX-License-Identifier: GPL-3.0-or-later
|
|
|
|
package cutover
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
const realisticUnit = `[Unit]
|
|
Description=Stalwart Mail Server
|
|
After=network.target
|
|
|
|
[Service]
|
|
Type=simple
|
|
User=stalwart
|
|
ExecStart=/usr/local/bin/stalwart --config /etc/stalwart/config.toml
|
|
Restart=on-failure
|
|
LimitNOFILE=65536
|
|
ProtectSystem=strict
|
|
ReadWritePaths=/var/lib/stalwart
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
`
|
|
|
|
func TestRewriteUnitRepointsExecStart(t *testing.T) {
|
|
got, err := RewriteUnit(realisticUnit, "/usr/local/bin/stalwart", "/etc/stalwart/config.json")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(got, "ExecStart=/usr/local/bin/stalwart --config /etc/stalwart/config.json") {
|
|
t.Errorf("ExecStart not repointed:\n%s", got)
|
|
}
|
|
}
|
|
|
|
// The operator's unit is theirs: hardening options, limits and paths this
|
|
// tool has no opinion about must survive untouched.
|
|
func TestRewriteUnitPreservesEverythingElse(t *testing.T) {
|
|
got, err := RewriteUnit(realisticUnit, "/opt/stalwart/bin/stalwart", "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, want := range []string{
|
|
"Description=Stalwart Mail Server", "User=stalwart", "Restart=on-failure",
|
|
"LimitNOFILE=65536", "ProtectSystem=strict", "ReadWritePaths=/var/lib/stalwart",
|
|
"WantedBy=multi-user.target",
|
|
} {
|
|
if !strings.Contains(got, want) {
|
|
t.Errorf("rewrite dropped %q:\n%s", want, got)
|
|
}
|
|
}
|
|
if !strings.Contains(got, "ExecStart=/opt/stalwart/bin/stalwart --config /etc/stalwart/config.toml") {
|
|
t.Errorf("existing --config should be preserved when no new one is given:\n%s", got)
|
|
}
|
|
}
|
|
|
|
func TestRewriteUnitAddsConfigWhenTheUnitHasNone(t *testing.T) {
|
|
got, err := RewriteUnit("[Service]\nExecStart=/usr/local/bin/stalwart\n", "/usr/local/bin/stalwart", "/etc/stalwart/config.json")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(got, "ExecStart=/usr/local/bin/stalwart --config /etc/stalwart/config.json") {
|
|
t.Errorf("--config not added:\n%s", got)
|
|
}
|
|
}
|
|
|
|
func TestRewriteUnitKeepsSystemdExecPrefixes(t *testing.T) {
|
|
got, err := RewriteUnit("[Service]\nExecStart=-@/old/stalwart --config /c\n", "/new/stalwart", "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(got, "ExecStart=-@/new/stalwart --config /c") {
|
|
t.Errorf("systemd exec prefix characters were dropped, changing what the unit means:\n%s", got)
|
|
}
|
|
}
|
|
|
|
// Leaving STALWART_RECOVERY_MODE=1 in the unit is the documented footgun
|
|
// from §4.5: the service would recovery-boot on every restart, forever.
|
|
func TestRewriteUnitStripsRecoveryEnvironmentLines(t *testing.T) {
|
|
unit := `[Service]
|
|
Environment=STALWART_RECOVERY_MODE=1
|
|
Environment="STALWART_RECOVERY_ADMIN=admin:hunter2"
|
|
Environment=RUST_LOG=info
|
|
ExecStart=/usr/local/bin/stalwart
|
|
`
|
|
got, err := RewriteUnit(unit, "/usr/local/bin/stalwart", "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, gone := range []string{"STALWART_RECOVERY_MODE", "STALWART_RECOVERY_ADMIN"} {
|
|
if strings.Contains(got, gone) {
|
|
t.Errorf("%s survived the rewrite - the service would recovery-boot on every restart:\n%s", gone, got)
|
|
}
|
|
}
|
|
if !strings.Contains(got, "Environment=RUST_LOG=info") {
|
|
t.Errorf("unrelated Environment line was dropped:\n%s", got)
|
|
}
|
|
}
|
|
|
|
// A line this tool only partly understands is one it must not edit.
|
|
func TestRewriteUnitRefusesAMixedEnvironmentLine(t *testing.T) {
|
|
unit := "[Service]\nEnvironment=RUST_LOG=info STALWART_RECOVERY_MODE=1\nExecStart=/usr/local/bin/stalwart\n"
|
|
_, err := RewriteUnit(unit, "/usr/local/bin/stalwart", "")
|
|
if err == nil {
|
|
t.Fatal("want refusal for an Environment line mixing recovery and other variables, got nil")
|
|
}
|
|
if !strings.Contains(err.Error(), "by hand") {
|
|
t.Errorf("error %q should tell the operator what to do about it", err)
|
|
}
|
|
}
|
|
|
|
func TestRewriteUnitRefusesAUnitWithNoExecStart(t *testing.T) {
|
|
_, err := RewriteUnit("[Unit]\nDescription=Something else entirely\n", "/usr/local/bin/stalwart", "")
|
|
if err == nil {
|
|
t.Fatal("want refusal for a unit with no ExecStart, got nil")
|
|
}
|
|
if !strings.Contains(err.Error(), "right unit file") {
|
|
t.Errorf("error %q should question whether this is the right file", err)
|
|
}
|
|
}
|
|
|
|
func TestRewriteUnitHandlesMultipleExecStartLines(t *testing.T) {
|
|
unit := "[Service]\nExecStart=\nExecStart=/old/stalwart --config /c\n"
|
|
got, err := RewriteUnit(unit, "/new/stalwart", "")
|
|
if err == nil {
|
|
t.Fatalf("an empty ExecStart= names no executable and should be refused, got:\n%s", got)
|
|
}
|
|
}
|
|
|
|
// A real production unit writes ExecStart=... --config=/path. Only matching
|
|
// the separated "--config /path" form appended a second flag, leaving the
|
|
// service started with two configs and using the v0.15 one - which v0.16
|
|
// cannot read as a store descriptor. Found while preparing a live
|
|
// migration, before it ran.
|
|
func TestRewriteUnitReplacesTheEqualsFormConfig(t *testing.T) {
|
|
unit := "[Service]\nExecStart=/opt/stalwart/bin/stalwart --config=/opt/stalwart/etc/config.toml\n"
|
|
got, err := RewriteUnit(unit, "/opt/stalwart/bin/stalwart", "/opt/stalwart/etc/config.json")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Count(got, "--config") != 1 {
|
|
t.Errorf("expected exactly one --config argument, got:\n%s", got)
|
|
}
|
|
if !strings.Contains(got, "--config=/opt/stalwart/etc/config.json") {
|
|
t.Errorf("equals-form config not replaced:\n%s", got)
|
|
}
|
|
if strings.Contains(got, "config.toml") {
|
|
t.Errorf("the old config path survived:\n%s", got)
|
|
}
|
|
}
|
|
|
|
// The separated form must keep working; both spellings are real.
|
|
func TestRewriteUnitReplacesTheSeparatedFormConfig(t *testing.T) {
|
|
unit := "[Service]\nExecStart=/usr/local/bin/stalwart --config /etc/stalwart/config.toml\n"
|
|
got, err := RewriteUnit(unit, "/usr/local/bin/stalwart", "/etc/stalwart/config.json")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Count(got, "--config") != 1 {
|
|
t.Errorf("expected exactly one --config argument, got:\n%s", got)
|
|
}
|
|
if !strings.Contains(got, "--config /etc/stalwart/config.json") {
|
|
t.Errorf("separated-form config not replaced:\n%s", got)
|
|
}
|
|
}
|