stalwart-migrator/internal/cutover/cutover_test.go

package cutover

import (
	"context"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/LINUXexpert-org/stalwart-migrator/internal/checkpoint"
	"github.com/LINUXexpert-org/stalwart-migrator/internal/service"
)

// fakeController stands in for systemd, recording call order.
type fakeController struct {
	calls    []string
	active   bool
	startErr error
}

func (f *fakeController) Stop(context.Context) error {
	f.calls = append(f.calls, "stop")
	f.active = false
	return nil
}

func (f *fakeController) Start(context.Context) error {
	f.calls = append(f.calls, "start")
	if f.startErr != nil {
		return f.startErr
	}
	f.active = true
	return nil
}

func (f *fakeController) Active(context.Context) (bool, error) { return f.active, nil }

func (f *fakeController) ReloadConfig(context.Context) error {
	f.calls = append(f.calls, "reload")
	return nil
}

func (f *fakeController) Target() string { return "test service" }

func readFile(t *testing.T, path string) string {
	t.Helper()
	data, err := os.ReadFile(path)
	if os.IsNotExist(err) {
		return ""
	}
	if err != nil {
		t.Fatal(err)
	}
	return string(data)
}

// migratedRun builds a checkpoint for a run that backed up successfully and
// is ready to cut over: the state this phase is actually invoked against.
func migratedRun(t *testing.T) (store *checkpoint.Store, rs *checkpoint.RunState, opts Options) {
	t.Helper()
	root := t.TempDir()

	store = checkpoint.NewStore(filepath.Join(root, "runs"))
	rs, err := store.Create("0.15.5", "0.16.14")
	if err != nil {
		t.Fatal(err)
	}
	rs.Topology = checkpoint.Topology{DeploymentKind: "systemd", StoreBackend: "rocksdb"}

	staged := filepath.Join(root, "staged-stalwart")
	if err := os.WriteFile(staged, []byte("#!/bin/sh\necho 'stalwart 0.16.14'\n"), 0o755); err != nil {
		t.Fatal(err)
	}
	unitPath := filepath.Join(root, "stalwart.service")
	if err := os.WriteFile(unitPath, []byte(realisticUnit), 0o644); err != nil {
		t.Fatal(err)
	}

	return store, rs, Options{
		StagedBinaryPath:       staged,
		BinaryPath:             filepath.Join(root, "bin-stalwart"),
		ServiceUnitPath:        unitPath,
		RecoveryPointConfirmed: true,
		Controller:             &fakeController{},
	}
}

// This tool can't undo a cutover, so the least it can do is refuse to
// perform one without the operator having been asked the question.
func TestBuildPlanRefusesWithoutAConfirmedRecoveryPoint(t *testing.T) {
	_, rs, opts := migratedRun(t)
	opts.RecoveryPointConfirmed = false

	_, err := BuildPlan(rs, opts)
	if err == nil {
		t.Fatal("BuildPlan: want refusal when no recovery point has been confirmed, got nil")
	}
	if !strings.Contains(err.Error(), "no way to undo") {
		t.Errorf("error %q should be plain that this is irreversible for the tool", err)
	}
}

func TestBuildPlanRefusesDockerDeployments(t *testing.T) {
	_, rs, opts := migratedRun(t)
	rs.Topology.DeploymentKind = string(service.Docker)
	opts.Controller = nil

	_, err := BuildPlan(rs, opts)
	if err == nil {
		t.Fatal("BuildPlan: want refusal for a container deployment, got nil")
	}
	if !strings.Contains(err.Error(), "recreating the container") {
		t.Errorf("error %q should explain what cutting over a container would actually involve", err)
	}
}

func TestBuildPlanRefusesAMissingStagedBinaryOrUnit(t *testing.T) {
	for _, tc := range []struct{ name, field string }{{"staged binary", "staged"}, {"service unit", "unit"}} {
		t.Run(tc.name, func(t *testing.T) {
			_, rs, opts := migratedRun(t)
			if tc.field == "staged" {
				opts.StagedBinaryPath = filepath.Join(t.TempDir(), "absent")
			} else {
				opts.ServiceUnitPath = filepath.Join(t.TempDir(), "absent.service")
			}
			if _, err := BuildPlan(rs, opts); err == nil {
				t.Fatalf("BuildPlan: want refusal for a missing %s, got nil", tc.name)
			}
		})
	}
}

func TestRunInstallsRewritesAndStarts(t *testing.T) {
	store, rs, opts := migratedRun(t)
	ctl := opts.Controller.(*fakeController)

	report, err := Run(context.Background(), store, rs, opts)
	if err != nil {
		t.Fatalf("Run: %v\n%s", err, report)
	}

	if got := readFile(t, opts.BinaryPath); !strings.Contains(got, "0.16.14") {
		t.Errorf("installed binary = %q, want the staged one", got)
	}
	if info, err := os.Stat(opts.BinaryPath); err != nil || info.Mode().Perm() != 0o755 {
		t.Errorf("installed binary mode = %v (err %v), want 0755 - a non-executable binary won't start", info.Mode().Perm(), err)
	}
	unit := readFile(t, opts.ServiceUnitPath)
	if !strings.Contains(unit, "ExecStart="+opts.BinaryPath) {
		t.Errorf("unit not repointed at the new binary:\n%s", unit)
	}
	if got, want := strings.Join(ctl.calls, ","), "reload,start"; got != want {
		t.Errorf("controller calls = %q, want %q - the definition must be reloaded before the start", got, want)
	}
	if report.Blocking() {
		t.Errorf("report should be clean:\n%s", report)
	}
}

// Recovery is the operator's own snapshot, but a snapshot revert doesn't
// help someone who only wants their unit file back - so this phase has to
// leave the original where they can find it.
func TestRunPreservesTheOriginalServiceDefinition(t *testing.T) {
	store, rs, opts := migratedRun(t)
	if _, err := Run(context.Background(), store, rs, opts); err != nil {
		t.Fatal(err)
	}

	art, found := rs.Artifacts[ArtifactServiceUnit]
	if !found {
		t.Fatalf("no %q artifact recorded; an operator restoring by hand would have to reconstruct the unit from memory", ArtifactServiceUnit)
	}
	preserved := readFile(t, art.Path)
	if !strings.Contains(preserved, "ExecStart=/usr/local/bin/stalwart --config /etc/stalwart/config.toml") {
		t.Errorf("preserved unit = %q, want the definition as it was before the rewrite", preserved)
	}
	if art.SHA256 == "" {
		t.Error("preserved unit artifact has no checksum")
	}
}

// Installing a binary that isn't the version the run planned for would
// migrate to a version nobody chose.
func TestRunRefusesAStagedBinaryOfTheWrongVersion(t *testing.T) {
	store, rs, opts := migratedRun(t)
	if err := os.WriteFile(opts.StagedBinaryPath, []byte("#!/bin/sh\necho 'stalwart 0.16.9'\n"), 0o755); err != nil {
		t.Fatal(err)
	}

	report, err := Run(context.Background(), store, rs, opts)
	if err == nil {
		t.Fatal("Run: want failure for a staged binary of the wrong version, got nil")
	}
	if _, statErr := os.Stat(opts.BinaryPath); !os.IsNotExist(statErr) {
		t.Error("the wrong-version binary was installed anyway")
	}
	if got := readFile(t, opts.ServiceUnitPath); !strings.Contains(got, "/usr/local/bin/stalwart") {
		t.Error("the service definition was rewritten despite the refusal")
	}
	if !report.Blocking() {
		t.Error("report should be blocking")
	}
}

func TestRunResumesWithoutRedoingCompletedSteps(t *testing.T) {
	store, rs, opts := migratedRun(t)
	if _, err := Run(context.Background(), store, rs, opts); err != nil {
		t.Fatal(err)
	}

	second := &fakeController{active: true}
	opts.Controller = second
	reloaded, err := store.Load(rs.RunID)
	if err != nil {
		t.Fatal(err)
	}
	if _, err := Run(context.Background(), store, reloaded, opts); err != nil {
		t.Fatal(err)
	}
	if len(second.calls) != 0 {
		t.Errorf("second invocation called the controller %v, want nothing - every step was already done", second.calls)
	}
}

func TestRunFailsWhenTheMigratedServiceNeverAnswers(t *testing.T) {
	store, rs, opts := migratedRun(t)
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusBadGateway)
	}))
	defer srv.Close()
	opts.AdminURL = srv.URL
	opts.HealthTimeout = 300 * time.Millisecond

	report, err := Run(context.Background(), store, rs, opts)
	if err == nil {
		t.Fatal("Run: want failure when the started service never answers, got nil")
	}
	if !strings.Contains(err.Error(), "never answered") {
		t.Errorf("error %q should distinguish 'started but not answering' from 'failed to start'", err)
	}
	if !report.Blocking() {
		t.Error("report should be blocking")
	}
}

// Rolling back a migration that completed successfully, because a counter
// didn't get rebuilt, would be worse than a stale counter.
func TestRunWarnsRatherThanFailsWhenQuotaRecalculationFails(t *testing.T) {
	store, rs, opts := migratedRun(t)
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Method == http.MethodGet {
			json.NewEncoder(w).Encode(map[string]any{"apiUrl": "/api"})
			return
		}
		w.WriteHeader(http.StatusInternalServerError) // the management API is unhappy
	}))
	defer srv.Close()
	opts.AdminURL = srv.URL
	opts.RecalculateQuotas = true

	report, err := Run(context.Background(), store, rs, opts)
	if err != nil {
		t.Fatalf("a failed quota rebuild must not fail the cutover: %v\n%s", err, report)
	}
	if report.Blocking() {
		t.Errorf("report should not be blocking:\n%s", report)
	}
	var warned bool
	for _, res := range report.Results {
		if res.Name == "recalculate-quotas" {
			warned = res.Status == StatusWarn
			if !strings.Contains(res.Detail, "Tasks panel") {
				t.Errorf("warning %q should tell the operator how to finish it by hand", res.Detail)
			}
		}
	}
	if !warned {
		t.Errorf("quota failure should be a warning, not silence:\n%s", report)
	}
}

func TestRunSkipsQuotaRecalculationOnThePatchFastPath(t *testing.T) {
	store, rs, opts := migratedRun(t)
	opts.RecalculateQuotas = false

	report, err := Run(context.Background(), store, rs, opts)
	if err != nil {
		t.Fatal(err)
	}
	for _, res := range report.Results {
		if res.Name == "recalculate-quotas" {
			if res.Status != StatusSkipped {
				t.Errorf("recalculate-quotas = %s, want skip", res.Status)
			}
			if !strings.Contains(res.Detail, "0.15/0.16") {
				t.Errorf("skip detail %q should say why it isn't needed", res.Detail)
			}
		}
	}
}

// quotaServer answers the three calls quota recalculation makes: enumerate
// accounts, schedule one task per account, then poll until the queue
// drains. Tasks are removed once fetched, modelling a queue whose entries
// are consumed when they run.
func quotaServer(t *testing.T, accountIDs []string) (*httptest.Server, *[]map[string]any) {
	t.Helper()
	var scheduled []map[string]any
	queued := map[string]bool{}

	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Method == http.MethodGet {
			json.NewEncoder(w).Encode(map[string]any{"apiUrl": "/api"})
			return
		}
		var body map[string]any
		json.NewDecoder(r.Body).Decode(&body)
		call := body["methodCalls"].([]any)[0].([]any)
		name := call[0].(string)
		args := call[1].(map[string]any)

		switch name {
		case "x:Account/query":
			json.NewEncoder(w).Encode(map[string]any{"methodResponses": []any{
				[]any{"x:Account/query", map[string]any{"ids": accountIDs}, "q"},
			}})
		case "x:Task/set":
			created := map[string]any{}
			for creationID, obj := range args["create"].(map[string]any) {
				scheduled = append(scheduled, obj.(map[string]any))
				id := "task-" + creationID
				queued[id] = true
				created[creationID] = map[string]any{"id": id}
			}
			json.NewEncoder(w).Encode(map[string]any{"methodResponses": []any{
				[]any{"x:Task/set", map[string]any{"created": created}, "s"},
			}})
		case "x:Task/get":
			// Report every task as gone: it ran and left the queue.
			for _, raw := range args["ids"].([]any) {
				delete(queued, raw.(string))
			}
			json.NewEncoder(w).Encode(map[string]any{"methodResponses": []any{
				[]any{"x:Task/get", map[string]any{"list": []any{}}, "g"},
			}})
		default:
			t.Errorf("unexpected method call %s", name)
		}
	}))
	t.Cleanup(srv.Close)
	return srv, &scheduled
}

func TestRunSchedulesOneQuotaTaskPerAccountAndWaits(t *testing.T) {
	store, rs, opts := migratedRun(t)
	srv, scheduled := quotaServer(t, []string{"a1", "a2", "a3"})
	opts.AdminURL = srv.URL
	opts.AdminUser = "admin"
	opts.RecalculateQuotas = true

	report, err := Run(context.Background(), store, rs, opts)
	if err != nil {
		t.Fatalf("Run: %v\n%s", err, report)
	}
	if len(*scheduled) != 3 {
		t.Fatalf("scheduled %d task(s), want one per account", len(*scheduled))
	}
	for _, obj := range *scheduled {
		if obj["maintenanceType"] != "recalculateQuota" {
			t.Errorf("maintenanceType = %v, want recalculateQuota", obj["maintenanceType"])
		}
	}
	for _, res := range report.Results {
		if res.Name == "recalculate-quotas" {
			if res.Status != StatusOK {
				t.Errorf("recalculate-quotas = %s: %s", res.Status, res.Detail)
			}
			if !strings.Contains(res.Detail, "3 account(s)") {
				t.Errorf("detail %q should say how many accounts were rebuilt", res.Detail)
			}
		}
	}
}