Files
stalwart-migrator/internal/cutover/cutover_test.go
T
jcoffey-dev 3e155fa42c Fix three defects a full VM migration exposed
Ran a complete 0.15.5 -> 0.16.14 migration of the smoke VM, driving the
phases in the order the real pipeline will. It worked - all mail intact and
readable afterwards, all ten listeners up, cutover executed for the first
time ever and checkpoint resume exercised - and it exposed three defects.

1. The converted config was installed root-owned while the service runs as
   its own user. Stalwart crash-looped 28 times on "Failed to read data
   store settings: Permission denied", minutes after the mistake and
   nowhere near it. This is the same ownership trap that retired the
   rollback implementation, in a new place: writing files as root is the
   natural thing for a tool running as root to do, and it is wrong every
   time the service is not root.

   Cutover now installs the config itself, copying ownership and mode from
   the config being replaced.

2. v0.16.14 does not serve /api - the endpoint stalwartapi assumed.
   Confirmed against a fully migrated, fully configured, serving instance
   rather than a sandbox: /api, /api/principal and /jmap/ all 404. The JMAP
   endpoint is the one the session document advertises, which is what RFC
   8620 discovery is for.

   The client now discovers it, re-basing the advertised path onto the
   operator's host: a real instance advertises its canonical public URL
   ("https://mail.smoke.test/jmap/") which frequently isn't reachable from
   where this tool runs. The session is authoritative about the path; the
   operator is authoritative about the host.

3. Dispatching on the urn:stalwart:jmap capability was wrong, because
   NEITHER version advertises it - not 0.15.5, and not a fully migrated
   0.16.14. That sent 0.16 instances down the 0.15 REST path where every
   call 404s. The client probes what the instance actually serves instead.
   Less elegant than a declared capability, with the advantage of being
   true.

Also: a JMAP "forbidden" now explains itself. An account holding the admin
role before the migration was refused x:Account/query afterwards, and a
bare "forbidden" gives an operator nowhere to start. Whether the role
failed to carry or v0.16 wants different permissions was not isolated, and
that question is recorded as open - it gates quota recalculation and any
post-migration validation.

Verified against both live instances: the 0.15.5 reports 3 accounts and its
domain over REST, and the migrated 0.16.14 routes to JMAP, finds the right
endpoint, and returns the explained refusal.
2026-08-23 21:32:30 -07:00

451 lines
15 KiB
Go

// SPDX-FileCopyrightText: 2026 LINUXexpert-org
// SPDX-License-Identifier: GPL-3.0-or-later
package cutover
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/LINUXexpert-org/stalwart-migrator/internal/checkpoint"
"github.com/LINUXexpert-org/stalwart-migrator/internal/service"
)
// fakeController stands in for systemd, recording call order.
type fakeController struct {
calls []string
active bool
startErr error
}
func (f *fakeController) Stop(context.Context) error {
f.calls = append(f.calls, "stop")
f.active = false
return nil
}
func (f *fakeController) Start(context.Context) error {
f.calls = append(f.calls, "start")
if f.startErr != nil {
return f.startErr
}
f.active = true
return nil
}
func (f *fakeController) Active(context.Context) (bool, error) { return f.active, nil }
func (f *fakeController) ReloadConfig(context.Context) error {
f.calls = append(f.calls, "reload")
return nil
}
func (f *fakeController) Target() string { return "test service" }
func readFile(t *testing.T, path string) string {
t.Helper()
data, err := os.ReadFile(path)
if os.IsNotExist(err) {
return ""
}
if err != nil {
t.Fatal(err)
}
return string(data)
}
// migratedRun builds a checkpoint for a run that backed up successfully and
// is ready to cut over: the state this phase is actually invoked against.
func migratedRun(t *testing.T) (store *checkpoint.Store, rs *checkpoint.RunState, opts Options) {
t.Helper()
root := t.TempDir()
store = checkpoint.NewStore(filepath.Join(root, "runs"))
rs, err := store.Create("0.15.5", "0.16.14")
if err != nil {
t.Fatal(err)
}
rs.Topology = checkpoint.Topology{DeploymentKind: "systemd", StoreBackend: "rocksdb"}
staged := filepath.Join(root, "staged-stalwart")
if err := os.WriteFile(staged, []byte("#!/bin/sh\necho 'stalwart 0.16.14'\n"), 0o755); err != nil {
t.Fatal(err)
}
unitPath := filepath.Join(root, "stalwart.service")
if err := os.WriteFile(unitPath, []byte(realisticUnit), 0o644); err != nil {
t.Fatal(err)
}
return store, rs, Options{
StagedBinaryPath: staged,
BinaryPath: filepath.Join(root, "bin-stalwart"),
ServiceUnitPath: unitPath,
RecoveryPointConfirmed: true,
Controller: &fakeController{},
}
}
// This tool can't undo a cutover, so the least it can do is refuse to
// perform one without the operator having been asked the question.
func TestBuildPlanRefusesWithoutAConfirmedRecoveryPoint(t *testing.T) {
_, rs, opts := migratedRun(t)
opts.RecoveryPointConfirmed = false
_, err := BuildPlan(rs, opts)
if err == nil {
t.Fatal("BuildPlan: want refusal when no recovery point has been confirmed, got nil")
}
if !strings.Contains(err.Error(), "no way to undo") {
t.Errorf("error %q should be plain that this is irreversible for the tool", err)
}
}
func TestBuildPlanRefusesDockerDeployments(t *testing.T) {
_, rs, opts := migratedRun(t)
rs.Topology.DeploymentKind = string(service.Docker)
opts.Controller = nil
_, err := BuildPlan(rs, opts)
if err == nil {
t.Fatal("BuildPlan: want refusal for a container deployment, got nil")
}
if !strings.Contains(err.Error(), "recreating the container") {
t.Errorf("error %q should explain what cutting over a container would actually involve", err)
}
}
func TestBuildPlanRefusesAMissingStagedBinaryOrUnit(t *testing.T) {
for _, tc := range []struct{ name, field string }{{"staged binary", "staged"}, {"service unit", "unit"}} {
t.Run(tc.name, func(t *testing.T) {
_, rs, opts := migratedRun(t)
if tc.field == "staged" {
opts.StagedBinaryPath = filepath.Join(t.TempDir(), "absent")
} else {
opts.ServiceUnitPath = filepath.Join(t.TempDir(), "absent.service")
}
if _, err := BuildPlan(rs, opts); err == nil {
t.Fatalf("BuildPlan: want refusal for a missing %s, got nil", tc.name)
}
})
}
}
func TestRunInstallsRewritesAndStarts(t *testing.T) {
store, rs, opts := migratedRun(t)
ctl := opts.Controller.(*fakeController)
report, err := Run(context.Background(), store, rs, opts)
if err != nil {
t.Fatalf("Run: %v\n%s", err, report)
}
if got := readFile(t, opts.BinaryPath); !strings.Contains(got, "0.16.14") {
t.Errorf("installed binary = %q, want the staged one", got)
}
if info, err := os.Stat(opts.BinaryPath); err != nil || info.Mode().Perm() != 0o755 {
t.Errorf("installed binary mode = %v (err %v), want 0755 - a non-executable binary won't start", info.Mode().Perm(), err)
}
unit := readFile(t, opts.ServiceUnitPath)
if !strings.Contains(unit, "ExecStart="+opts.BinaryPath) {
t.Errorf("unit not repointed at the new binary:\n%s", unit)
}
if got, want := strings.Join(ctl.calls, ","), "reload,start"; got != want {
t.Errorf("controller calls = %q, want %q - the definition must be reloaded before the start", got, want)
}
if report.Blocking() {
t.Errorf("report should be clean:\n%s", report)
}
}
// Recovery is the operator's own snapshot, but a snapshot revert doesn't
// help someone who only wants their unit file back - so this phase has to
// leave the original where they can find it.
func TestRunPreservesTheOriginalServiceDefinition(t *testing.T) {
store, rs, opts := migratedRun(t)
if _, err := Run(context.Background(), store, rs, opts); err != nil {
t.Fatal(err)
}
art, found := rs.Artifacts[ArtifactServiceUnit]
if !found {
t.Fatalf("no %q artifact recorded; an operator restoring by hand would have to reconstruct the unit from memory", ArtifactServiceUnit)
}
preserved := readFile(t, art.Path)
if !strings.Contains(preserved, "ExecStart=/usr/local/bin/stalwart --config /etc/stalwart/config.toml") {
t.Errorf("preserved unit = %q, want the definition as it was before the rewrite", preserved)
}
if art.SHA256 == "" {
t.Error("preserved unit artifact has no checksum")
}
}
// Installing a binary that isn't the version the run planned for would
// migrate to a version nobody chose.
func TestRunRefusesAStagedBinaryOfTheWrongVersion(t *testing.T) {
store, rs, opts := migratedRun(t)
if err := os.WriteFile(opts.StagedBinaryPath, []byte("#!/bin/sh\necho 'stalwart 0.16.9'\n"), 0o755); err != nil {
t.Fatal(err)
}
report, err := Run(context.Background(), store, rs, opts)
if err == nil {
t.Fatal("Run: want failure for a staged binary of the wrong version, got nil")
}
if _, statErr := os.Stat(opts.BinaryPath); !os.IsNotExist(statErr) {
t.Error("the wrong-version binary was installed anyway")
}
if got := readFile(t, opts.ServiceUnitPath); !strings.Contains(got, "/usr/local/bin/stalwart") {
t.Error("the service definition was rewritten despite the refusal")
}
if !report.Blocking() {
t.Error("report should be blocking")
}
}
func TestRunResumesWithoutRedoingCompletedSteps(t *testing.T) {
store, rs, opts := migratedRun(t)
if _, err := Run(context.Background(), store, rs, opts); err != nil {
t.Fatal(err)
}
second := &fakeController{active: true}
opts.Controller = second
reloaded, err := store.Load(rs.RunID)
if err != nil {
t.Fatal(err)
}
if _, err := Run(context.Background(), store, reloaded, opts); err != nil {
t.Fatal(err)
}
if len(second.calls) != 0 {
t.Errorf("second invocation called the controller %v, want nothing - every step was already done", second.calls)
}
}
func TestRunFailsWhenTheMigratedServiceNeverAnswers(t *testing.T) {
store, rs, opts := migratedRun(t)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusBadGateway)
}))
defer srv.Close()
opts.AdminURL = srv.URL
opts.HealthTimeout = 300 * time.Millisecond
report, err := Run(context.Background(), store, rs, opts)
if err == nil {
t.Fatal("Run: want failure when the started service never answers, got nil")
}
if !strings.Contains(err.Error(), "never answered") {
t.Errorf("error %q should distinguish 'started but not answering' from 'failed to start'", err)
}
if !report.Blocking() {
t.Error("report should be blocking")
}
}
// Rolling back a migration that completed successfully, because a counter
// didn't get rebuilt, would be worse than a stale counter.
func TestRunWarnsRatherThanFailsWhenQuotaRecalculationFails(t *testing.T) {
store, rs, opts := migratedRun(t)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodGet {
json.NewEncoder(w).Encode(map[string]any{"apiUrl": "/api"})
return
}
w.WriteHeader(http.StatusInternalServerError) // the management API is unhappy
}))
defer srv.Close()
opts.AdminURL = srv.URL
opts.RecalculateQuotas = true
report, err := Run(context.Background(), store, rs, opts)
if err != nil {
t.Fatalf("a failed quota rebuild must not fail the cutover: %v\n%s", err, report)
}
if report.Blocking() {
t.Errorf("report should not be blocking:\n%s", report)
}
var warned bool
for _, res := range report.Results {
if res.Name == "recalculate-quotas" {
warned = res.Status == StatusWarn
if !strings.Contains(res.Detail, "Tasks panel") {
t.Errorf("warning %q should tell the operator how to finish it by hand", res.Detail)
}
}
}
if !warned {
t.Errorf("quota failure should be a warning, not silence:\n%s", report)
}
}
func TestRunSkipsQuotaRecalculationOnThePatchFastPath(t *testing.T) {
store, rs, opts := migratedRun(t)
opts.RecalculateQuotas = false
report, err := Run(context.Background(), store, rs, opts)
if err != nil {
t.Fatal(err)
}
for _, res := range report.Results {
if res.Name == "recalculate-quotas" {
if res.Status != StatusSkipped {
t.Errorf("recalculate-quotas = %s, want skip", res.Status)
}
if !strings.Contains(res.Detail, "0.15/0.16") {
t.Errorf("skip detail %q should say why it isn't needed", res.Detail)
}
}
}
}
// quotaServer answers the three calls quota recalculation makes: enumerate
// accounts, schedule one task per account, then poll until the queue
// drains. Tasks are removed once fetched, modelling a queue whose entries
// are consumed when they run.
func quotaServer(t *testing.T, accountIDs []string) (*httptest.Server, *[]map[string]any) {
t.Helper()
var scheduled []map[string]any
queued := map[string]bool{}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodGet {
json.NewEncoder(w).Encode(map[string]any{"apiUrl": "/api"})
return
}
var body map[string]any
json.NewDecoder(r.Body).Decode(&body)
call := body["methodCalls"].([]any)[0].([]any)
name := call[0].(string)
args := call[1].(map[string]any)
switch name {
case "x:Account/query":
json.NewEncoder(w).Encode(map[string]any{"methodResponses": []any{
[]any{"x:Account/query", map[string]any{"ids": accountIDs}, "q"},
}})
case "x:Task/set":
created := map[string]any{}
for creationID, obj := range args["create"].(map[string]any) {
scheduled = append(scheduled, obj.(map[string]any))
id := "task-" + creationID
queued[id] = true
created[creationID] = map[string]any{"id": id}
}
json.NewEncoder(w).Encode(map[string]any{"methodResponses": []any{
[]any{"x:Task/set", map[string]any{"created": created}, "s"},
}})
case "x:Task/get":
// Report every task as gone: it ran and left the queue.
for _, raw := range args["ids"].([]any) {
delete(queued, raw.(string))
}
json.NewEncoder(w).Encode(map[string]any{"methodResponses": []any{
[]any{"x:Task/get", map[string]any{"list": []any{}}, "g"},
}})
default:
t.Errorf("unexpected method call %s", name)
}
}))
t.Cleanup(srv.Close)
return srv, &scheduled
}
func TestRunSchedulesOneQuotaTaskPerAccountAndWaits(t *testing.T) {
store, rs, opts := migratedRun(t)
srv, scheduled := quotaServer(t, []string{"a1", "a2", "a3"})
opts.AdminURL = srv.URL
opts.AdminUser = "admin"
opts.RecalculateQuotas = true
report, err := Run(context.Background(), store, rs, opts)
if err != nil {
t.Fatalf("Run: %v\n%s", err, report)
}
if len(*scheduled) != 3 {
t.Fatalf("scheduled %d task(s), want one per account", len(*scheduled))
}
for _, obj := range *scheduled {
if obj["maintenanceType"] != "recalculateQuota" {
t.Errorf("maintenanceType = %v, want recalculateQuota", obj["maintenanceType"])
}
}
for _, res := range report.Results {
if res.Name == "recalculate-quotas" {
if res.Status != StatusOK {
t.Errorf("recalculate-quotas = %s: %s", res.Status, res.Detail)
}
if !strings.Contains(res.Detail, "3 account(s)") {
t.Errorf("detail %q should say how many accounts were rebuilt", res.Detail)
}
}
}
}
// A full migration crash-looped 28 times on "Failed to read data store
// settings: Permission denied" because the converted config was written as
// root while the service runs as its own user. The failure surfaced minutes
// later in the journal, not at the moment of the mistake, which is what
// makes it worth a test rather than care.
func TestRunInstallsTheConfigWithOwnershipTheServiceCanRead(t *testing.T) {
store, rs, opts := migratedRun(t)
dir := t.TempDir()
// The config being replaced, standing in for the one the old version
// ran with - restrictive mode, so a naive copy would lock the service
// out of its own config.
oldConfig := filepath.Join(dir, "config.toml")
if err := os.WriteFile(oldConfig, []byte("[server]\n"), 0o640); err != nil {
t.Fatal(err)
}
converted := filepath.Join(dir, "converted.json")
if err := os.WriteFile(converted, []byte(`{"@type":"RocksDb","path":"/opt/stalwart/data"}`), 0o600); err != nil {
t.Fatal(err)
}
installed := filepath.Join(dir, "config.json")
opts.ConfigSource = converted
opts.ConfigPath = installed
opts.ConfigOwnerReference = oldConfig
report, err := Run(context.Background(), store, rs, opts)
if err != nil {
t.Fatalf("Run: %v\n%s", err, report)
}
if got := readFile(t, installed); !strings.Contains(got, "RocksDb") {
t.Errorf("installed config = %q, want the converted one", got)
}
info, err := os.Stat(installed)
if err != nil {
t.Fatal(err)
}
// Mode comes from the file being replaced, not from the source's 0600.
if info.Mode().Perm() != 0o640 {
t.Errorf("installed config mode = %v, want 0640 copied from the old config", info.Mode().Perm())
}
// The unit must point at the installed config, not the scratch copy.
if unit := readFile(t, opts.ServiceUnitPath); !strings.Contains(unit, installed) {
t.Errorf("unit does not reference the installed config:\n%s", unit)
}
}
func TestRunSkipsConfigInstallWhenThereIsNothingToInstall(t *testing.T) {
store, rs, opts := migratedRun(t)
report, err := Run(context.Background(), store, rs, opts)
if err != nil {
t.Fatal(err)
}
for _, res := range report.Results {
if res.Name == "install-config" && res.Status != StatusSkipped {
t.Errorf("install-config = %s, want skip when no ConfigSource is given", res.Status)
}
}
}