Initial commit: stalwart-migrator design and scaffolding

In-place upgrade tool for Stalwart Mail Server (0.15.5 -> latest) with
checkpointed rollback and post-migration validation. Design stage; see
ARCHITECTURE.md.

internal/checkpoint/doc.go

@@ -0,0 +1,3 @@
+// Package checkpoint implements run-id and state.json persistence, resume logic, and rollback-window tracking.
+// See ARCHITECTURE.md §5 for the design.
+package checkpoint

internal/checkpoint/step.go

@@ -0,0 +1,88 @@
+package checkpoint
+
+import (
+	"fmt"
+	"time"
+)
+
+func (rs *RunState) stepIndex(phase Phase, name string) int {
+	for i := range rs.Steps {
+		if rs.Steps[i].Phase == phase && rs.Steps[i].Name == name {
+			return i
+		}
+	}
+	return -1
+}
+
+// Status returns the current status of a step, or StepPending if it has
+// never been started.
+func (rs *RunState) Status(phase Phase, name string) StepStatus {
+	if i := rs.stepIndex(phase, name); i >= 0 {
+		return rs.Steps[i].Status
+	}
+	return StepPending
+}
+
+// Done reports whether a step already completed successfully. Callers use
+// this to decide whether to skip work on resume.
+func (rs *RunState) Done(phase Phase, name string) bool {
+	return rs.Status(phase, name) == StepDone
+}
+
+// Outcome returns the StepOutcome recorded for a step, zero-valued if none.
+// This is what lets a resumed run reconstruct a skipped step's result
+// without re-executing it.
+func (rs *RunState) Outcome(phase Phase, name string) StepOutcome {
+	if i := rs.stepIndex(phase, name); i >= 0 {
+		return rs.Steps[i].StepOutcome
+	}
+	return StepOutcome{}
+}
+
+// Begin marks a step as running, creating its record on first attempt or
+// resetting it on a retry after a prior failure.
+func (rs *RunState) Begin(phase Phase, name string) {
+	now := time.Now().UTC()
+	if i := rs.stepIndex(phase, name); i >= 0 {
+		rs.Steps[i].Status = StepRunning
+		rs.Steps[i].StartedAt = &now
+		rs.Steps[i].CompletedAt = nil
+		rs.Steps[i].StepOutcome = StepOutcome{}
+		rs.Steps[i].Error = ""
+	} else {
+		rs.Steps = append(rs.Steps, StepRecord{
+			Phase: phase, Name: name, Status: StepRunning, StartedAt: &now,
+		})
+	}
+	rs.UpdatedAt = now
+}
+
+// Complete marks a step done with its outcome. It panics if Begin was never
+// called for this step - that's a bug in the calling phase, not a runtime
+// condition callers should need to handle.
+func (rs *RunState) Complete(phase Phase, name string, outcome StepOutcome) {
+	i := rs.stepIndex(phase, name)
+	if i < 0 {
+		panic(fmt.Sprintf("checkpoint: Complete(%s/%s) called without Begin", phase, name))
+	}
+	now := time.Now().UTC()
+	rs.Steps[i].Status = StepDone
+	rs.Steps[i].CompletedAt = &now
+	rs.Steps[i].StepOutcome = outcome
+	rs.Steps[i].Error = ""
+	rs.UpdatedAt = now
+}
+
+// Fail marks a step failed, so a later Begin for the same (phase, name)
+// knows to retry it rather than treat it as done.
+func (rs *RunState) Fail(phase Phase, name string, stepErr error) {
+	i := rs.stepIndex(phase, name)
+	if i < 0 {
+		panic(fmt.Sprintf("checkpoint: Fail(%s/%s) called without Begin", phase, name))
+	}
+	now := time.Now().UTC()
+	rs.Steps[i].Status = StepFailed
+	rs.Steps[i].CompletedAt = &now
+	rs.Steps[i].Error = stepErr.Error()
+	rs.UpdatedAt = now
+}

internal/checkpoint/store.go

@@ -0,0 +1,179 @@
+package checkpoint
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"time"
+)
+
+// DefaultBaseDir is where runs are persisted when the operator doesn't
+// override it.
+const DefaultBaseDir = "/var/lib/stalwart-migrator/runs"
+
+// Store persists RunState to disk. It's the only thing in this package that
+// touches the filesystem - RunState itself is a plain data type.
+type Store struct {
+	baseDir string
+}
+
+func NewStore(baseDir string) *Store {
+	if baseDir == "" {
+		baseDir = DefaultBaseDir
+	}
+	return &Store{baseDir: baseDir}
+}
+
+func (s *Store) runDir(runID string) string    { return filepath.Join(s.baseDir, runID) }
+func (s *Store) statePath(runID string) string { return filepath.Join(s.runDir(runID), "state.json") }
+
+// Create starts a new run, assigns it an ID, and persists its initial
+// state before returning it.
+func (s *Store) Create(sourceVersion, targetVersion string) (*RunState, error) {
+	id, err := newRunID()
+	if err != nil {
+		return nil, fmt.Errorf("checkpoint: generate run id: %w", err)
+	}
+	now := time.Now().UTC()
+	rs := &RunState{
+		RunID:         id,
+		SourceVersion: sourceVersion,
+		TargetVersion: targetVersion,
+		CreatedAt:     now,
+		UpdatedAt:     now,
+		Artifacts:     map[string]Artifact{},
+	}
+	if err := s.Save(rs); err != nil {
+		return nil, err
+	}
+	return rs, nil
+}
+
+// Load reads an existing run's state from disk.
+func (s *Store) Load(runID string) (*RunState, error) {
+	data, err := os.ReadFile(s.statePath(runID))
+	if err != nil {
+		return nil, fmt.Errorf("checkpoint: load run %s: %w", runID, err)
+	}
+	var rs RunState
+	if err := json.Unmarshal(data, &rs); err != nil {
+		return nil, fmt.Errorf("checkpoint: parse run %s: %w", runID, err)
+	}
+	return &rs, nil
+}
+
+// Save writes state to disk atomically: write to a temp file in the same
+// directory, fsync it, then rename over the real path. A crash mid-write
+// leaves the temp file orphaned and state.json untouched, never a
+// truncated or corrupt state.json - that file is the one thing every phase
+// and a human operator both trust as the source of truth for what's
+// already happened, so a half-written version of it would be worse than an
+// old one.
+func (s *Store) Save(rs *RunState) error {
+	dir := s.runDir(rs.RunID)
+	if err := os.MkdirAll(dir, 0o750); err != nil {
+		return fmt.Errorf("checkpoint: create run directory: %w", err)
+	}
+	data, err := json.MarshalIndent(rs, "", "  ")
+	if err != nil {
+		return fmt.Errorf("checkpoint: marshal state: %w", err)
+	}
+	tmp, err := os.CreateTemp(dir, "state-*.json.tmp")
+	if err != nil {
+		return fmt.Errorf("checkpoint: create temp state file: %w", err)
+	}
+	tmpPath := tmp.Name()
+	if _, err := tmp.Write(data); err != nil {
+		tmp.Close()
+		os.Remove(tmpPath)
+		return fmt.Errorf("checkpoint: write temp state file: %w", err)
+	}
+	if err := tmp.Sync(); err != nil {
+		tmp.Close()
+		os.Remove(tmpPath)
+		return fmt.Errorf("checkpoint: sync temp state file: %w", err)
+	}
+	if err := tmp.Close(); err != nil {
+		os.Remove(tmpPath)
+		return fmt.Errorf("checkpoint: close temp state file: %w", err)
+	}
+	if err := os.Rename(tmpPath, s.statePath(rs.RunID)); err != nil {
+		os.Remove(tmpPath)
+		return fmt.Errorf("checkpoint: rename temp state file into place: %w", err)
+	}
+	return nil
+}
+
+// List returns known run IDs, most recently created first.
+func (s *Store) List() ([]string, error) {
+	entries, err := os.ReadDir(s.baseDir)
+	if os.IsNotExist(err) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("checkpoint: list runs: %w", err)
+	}
+	type run struct {
+		id      string
+		created time.Time
+	}
+	var runs []run
+	for _, e := range entries {
+		if !e.IsDir() {
+			continue
+		}
+		rs, err := s.Load(e.Name())
+		if err != nil {
+			continue // not a run directory (or a corrupt one) - skip it
+		}
+		runs = append(runs, run{rs.RunID, rs.CreatedAt})
+	}
+	sort.Slice(runs, func(i, j int) bool { return runs[i].created.After(runs[j].created) })
+	ids := make([]string, len(runs))
+	for i, r := range runs {
+		ids[i] = r.id
+	}
+	return ids, nil
+}
+
+// RunStep executes fn for (phase, name) unless it already completed
+// successfully in a prior attempt at this run, in which case fn is skipped
+// and the previously recorded StepOutcome is returned instead - this is
+// what makes an interrupted run resumable without redoing (or worse,
+// double-applying) work that already happened. State is persisted both
+// before fn runs (so a crash during fn is visible as "running", not silently
+// forgotten) and after (recording success or failure).
+func (s *Store) RunStep(rs *RunState, phase Phase, name string, fn func() (StepOutcome, error)) (StepOutcome, error) {
+	if rs.Done(phase, name) {
+		return rs.Outcome(phase, name), nil
+	}
+	rs.Begin(phase, name)
+	if err := s.Save(rs); err != nil {
+		return StepOutcome{}, fmt.Errorf("checkpoint: persist step start for %s/%s: %w", phase, name, err)
+	}
+	outcome, stepErr := fn()
+	if stepErr != nil {
+		rs.Fail(phase, name, stepErr)
+	} else {
+		rs.Complete(phase, name, outcome)
+	}
+	if err := s.Save(rs); err != nil {
+		if stepErr != nil {
+			return outcome, fmt.Errorf("%w (additionally failed to persist checkpoint: %v)", stepErr, err)
+		}
+		return outcome, fmt.Errorf("step %s/%s succeeded but failed to persist checkpoint: %w", phase, name, err)
+	}
+	return outcome, stepErr
+}
+
+func newRunID() (string, error) {
+	b := make([]byte, 4)
+	if _, err := rand.Read(b); err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("%s-%s", time.Now().UTC().Format("20060102-150405"), hex.EncodeToString(b)), nil
+}

internal/checkpoint/store_test.go

@@ -0,0 +1,168 @@
+package checkpoint
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestCreateSaveLoadRoundtrip(t *testing.T) {
+	store := NewStore(t.TempDir())
+
+	rs, err := store.Create("0.15.5", "0.16.14")
+	if err != nil {
+		t.Fatalf("Create: %v", err)
+	}
+	if rs.RunID == "" {
+		t.Fatal("Create: expected a non-empty run id")
+	}
+
+	rs.Topology = Topology{DeploymentKind: "systemd", StoreBackend: "rocksdb"}
+	rs.RecordArtifact("fs-backup", Artifact{Path: "/var/lib/stalwart.v0155-backup", SHA256: "deadbeef", SizeBytes: 1024})
+	if err := store.Save(rs); err != nil {
+		t.Fatalf("Save: %v", err)
+	}
+
+	loaded, err := store.Load(rs.RunID)
+	if err != nil {
+		t.Fatalf("Load: %v", err)
+	}
+	if loaded.SourceVersion != "0.15.5" || loaded.TargetVersion != "0.16.14" {
+		t.Errorf("Load: versions = %s -> %s, want 0.15.5 -> 0.16.14", loaded.SourceVersion, loaded.TargetVersion)
+	}
+	if loaded.Topology.DeploymentKind != "systemd" {
+		t.Errorf("Load: DeploymentKind = %q, want systemd", loaded.Topology.DeploymentKind)
+	}
+	if got := loaded.Artifacts["fs-backup"].SHA256; got != "deadbeef" {
+		t.Errorf("Load: artifact sha256 = %q, want deadbeef", got)
+	}
+}
+
+func TestSaveIsAtomicNoLeftoverTempFiles(t *testing.T) {
+	dir := t.TempDir()
+	store := NewStore(dir)
+	rs, err := store.Create("0.15.5", "0.16.14")
+	if err != nil {
+		t.Fatalf("Create: %v", err)
+	}
+
+	for i := 0; i < 5; i++ {
+		rs.Begin(PhasePreflight, "version")
+		rs.Complete(PhasePreflight, "version", StepOutcome{Verdict: "ok", Detail: "ok"})
+		if err := store.Save(rs); err != nil {
+			t.Fatalf("Save #%d: %v", i, err)
+		}
+	}
+
+	entries, err := os.ReadDir(filepath.Join(dir, rs.RunID))
+	if err != nil {
+		t.Fatalf("ReadDir: %v", err)
+	}
+	for _, e := range entries {
+		if e.Name() != "state.json" {
+			t.Errorf("unexpected leftover file in run dir: %s", e.Name())
+		}
+	}
+}
+
+func TestRunStepSkipsAlreadyDoneStep(t *testing.T) {
+	store := NewStore(t.TempDir())
+	rs, err := store.Create("0.15.5", "0.16.14")
+	if err != nil {
+		t.Fatalf("Create: %v", err)
+	}
+
+	calls := 0
+	fn := func() (StepOutcome, error) {
+		calls++
+		return StepOutcome{Verdict: "ok", Detail: "did the thing"}, nil
+	}
+
+	outcome1, err := store.RunStep(rs, PhasePreflight, "version", fn)
+	if err != nil {
+		t.Fatalf("RunStep #1: %v", err)
+	}
+	if outcome1.Detail != "did the thing" {
+		t.Errorf("RunStep #1 detail = %q, want %q", outcome1.Detail, "did the thing")
+	}
+	if calls != 1 {
+		t.Fatalf("calls after first RunStep = %d, want 1", calls)
+	}
+
+	// Simulate a resumed run: same rs, same step name. fn must not run again,
+	// and the previously recorded outcome must come back unchanged.
+	outcome2, err := store.RunStep(rs, PhasePreflight, "version", fn)
+	if err != nil {
+		t.Fatalf("RunStep #2 (resume): %v", err)
+	}
+	if calls != 1 {
+		t.Errorf("calls after resumed RunStep = %d, want 1 (fn should be skipped)", calls)
+	}
+	if outcome2 != outcome1 {
+		t.Errorf("RunStep #2 outcome = %+v, want %+v (unchanged from before)", outcome2, outcome1)
+	}
+}
+
+func TestRunStepRetriesAfterFailure(t *testing.T) {
+	store := NewStore(t.TempDir())
+	rs, err := store.Create("0.15.5", "0.16.14")
+	if err != nil {
+		t.Fatalf("Create: %v", err)
+	}
+
+	calls := 0
+	failThenSucceed := func() (StepOutcome, error) {
+		calls++
+		if calls == 1 {
+			return StepOutcome{}, errors.New("transient failure")
+		}
+		return StepOutcome{Verdict: "ok", Detail: "succeeded on retry"}, nil
+	}
+
+	if _, err := store.RunStep(rs, PhaseBackup, "fs-snapshot", failThenSucceed); err == nil {
+		t.Fatal("RunStep #1: expected error, got nil")
+	}
+	if rs.Status(PhaseBackup, "fs-snapshot") != StepFailed {
+		t.Errorf("status after failed attempt = %s, want failed", rs.Status(PhaseBackup, "fs-snapshot"))
+	}
+
+	outcome, err := store.RunStep(rs, PhaseBackup, "fs-snapshot", failThenSucceed)
+	if err != nil {
+		t.Fatalf("RunStep #2 (retry): %v", err)
+	}
+	if calls != 2 {
+		t.Errorf("calls = %d, want 2 (failed step must retry, not skip)", calls)
+	}
+	if outcome.Detail != "succeeded on retry" {
+		t.Errorf("detail = %q, want %q", outcome.Detail, "succeeded on retry")
+	}
+	if !rs.Done(PhaseBackup, "fs-snapshot") {
+		t.Error("step should be Done after a successful retry")
+	}
+}
+
+func TestListOrdersNewestFirst(t *testing.T) {
+	store := NewStore(t.TempDir())
+	first, err := store.Create("0.15.5", "0.16.14")
+	if err != nil {
+		t.Fatalf("Create first: %v", err)
+	}
+	first.CreatedAt = first.CreatedAt.Add(-time.Hour)
+	if err := store.Save(first); err != nil {
+		t.Fatalf("Save first: %v", err)
+	}
+	second, err := store.Create("0.16.14", "0.16.15")
+	if err != nil {
+		t.Fatalf("Create second: %v", err)
+	}
+
+	ids, err := store.List()
+	if err != nil {
+		t.Fatalf("List: %v", err)
+	}
+	if len(ids) != 2 || ids[0] != second.RunID || ids[1] != first.RunID {
+		t.Errorf("List = %v, want [%s %s]", ids, second.RunID, first.RunID)
+	}
+}

internal/checkpoint/types.go

@@ -0,0 +1,121 @@
+package checkpoint
+
+import "time"
+
+// Phase identifies one of the top-level migration phases from
+// ARCHITECTURE.md §4. Step records are scoped to a phase so the same step
+// name can be reused across phases without colliding.
+type Phase string
+
+const (
+	PhasePreflight Phase = "preflight"
+	PhaseBackup    Phase = "backup"
+	PhaseStage     Phase = "stage"
+	PhaseRecovery  Phase = "recovery"
+	PhaseCutover   Phase = "cutover"
+	PhaseValidate  Phase = "validate"
+	PhaseRollback  Phase = "rollback"
+)
+
+// StepStatus is the lifecycle state of one checkpointed step.
+type StepStatus string
+
+const (
+	StepPending StepStatus = "pending"
+	StepRunning StepStatus = "running"
+	StepDone    StepStatus = "done"
+	StepFailed  StepStatus = "failed"
+)
+
+// StepOutcome is what a step reports back on success: a verdict
+// classification the calling phase defines the meaning of (e.g. preflight's
+// "ok"/"warn"/"fail"), a human-readable summary, and an optional
+// machine-readable value later steps - or a resumed run reconstructing this
+// step's result without re-executing it - need. Keeping these three
+// separate (rather than one free-text field) is what lets `stalwart-migrate
+// status` print a clean human summary while still round-tripping the data a
+// resumed run depends on.
+type StepOutcome struct {
+	Verdict string `json:"verdict,omitempty"`
+	Detail  string `json:"detail,omitempty"`
+	Extra   string `json:"extra,omitempty"`
+}
+
+// StepRecord captures one step's lifecycle status plus its StepOutcome.
+type StepRecord struct {
+	Phase       Phase      `json:"phase"`
+	Name        string     `json:"name"`
+	Status      StepStatus `json:"status"`
+	StepOutcome            // embedded (untagged) so its fields flatten into this JSON object
+	StartedAt   *time.Time `json:"started_at,omitempty"`
+	CompletedAt *time.Time `json:"completed_at,omitempty"`
+	Error       string     `json:"error,omitempty"`
+}
+
+// Artifact is a content-addressed record of a file a run produced (a
+// backup, a settings dump, a downloaded release binary) so later phases and
+// a human operator can confirm it hasn't changed underfoot.
+type Artifact struct {
+	Path      string `json:"path"`
+	SHA256    string `json:"sha256"`
+	SizeBytes int64  `json:"size_bytes"`
+}
+
+// MailboxCount is one mailbox's message count as observed at preflight
+// time, for later comparison against the post-migration count.
+type MailboxCount struct {
+	Mailbox  string `json:"mailbox"`
+	Messages int    `json:"messages"`
+}
+
+// PreflightSnapshot holds the facts captured before anything is touched,
+// which the validate phase later compares against the migrated instance.
+// See ARCHITECTURE.md §4.1 and §4.7.
+type PreflightSnapshot struct {
+	TakenAt          time.Time                 `json:"taken_at"`
+	AccountCount     int                       `json:"account_count"`
+	Domains          []string                  `json:"domains,omitempty"`
+	MailboxCounts    map[string][]MailboxCount `json:"mailbox_counts,omitempty"` // account -> mailboxes
+	DKIMFingerprints map[string]string         `json:"dkim_fingerprints,omitempty"`
+	TLSFingerprints  []string                  `json:"tls_fingerprints,omitempty"`
+	ListenerPorts    []int                     `json:"listener_ports,omitempty"`
+}
+
+// Topology records how this Stalwart instance is deployed, as detected
+// during preflight, so later phases (cutover, rollback) know whether
+// they're managing a systemd unit or a container and what backend they're
+// dealing with.
+type Topology struct {
+	DeploymentKind string   `json:"deployment_kind,omitempty"` // "systemd", "docker", "unknown"
+	ClusterNodes   []string `json:"cluster_nodes,omitempty"`
+	StoreBackend   string   `json:"store_backend,omitempty"`
+	BlobStore      string   `json:"blob_store,omitempty"`
+	FTSBackend     string   `json:"fts_backend,omitempty"`
+}
+
+// RunState is the full persisted state of one migration run: everything
+// needed to resume it after a crash, decide whether to roll back, or
+// report on it later. See ARCHITECTURE.md §5.
+type RunState struct {
+	RunID                string              `json:"run_id"`
+	SourceVersion        string              `json:"source_version"`
+	TargetVersion        string              `json:"target_version"`
+	CreatedAt            time.Time           `json:"created_at"`
+	UpdatedAt            time.Time           `json:"updated_at"`
+	Topology             Topology            `json:"topology,omitempty"`
+	Steps                []StepRecord        `json:"steps"`
+	Artifacts            map[string]Artifact `json:"artifacts,omitempty"`
+	PreflightSnapshot    *PreflightSnapshot  `json:"preflight_snapshot,omitempty"`
+	RollbackWindowClosed bool                `json:"rollback_window_closed"`
+}
+
+// RecordArtifact stores a content-addressed record of a file this run
+// produced, keyed by a short logical name (e.g. "fs-backup", "settings-dump",
+// "target-binary") rather than its path, since the path alone doesn't prove
+// the content is what this run actually wrote.
+func (rs *RunState) RecordArtifact(name string, a Artifact) {
+	if rs.Artifacts == nil {
+		rs.Artifacts = map[string]Artifact{}
+	}
+	rs.Artifacts[name] = a
+}