Implement run: the migration pipeline, end to end

The phases have all existed for a while; nothing chained them. The order
here is the one arrived at by performing this migration by hand against a
clone of production before writing it down:

    preflight -> stage -> dump -> preserve binary -> STOP ->
    convert -> supplement -> recovery-mode migration -> cutover -> START

The dump runs before the stop because it reads settings over the admin API,
and a stopped server has no admin API. Everything from the stop to the end
of cutover is downtime.

internal/stage fills the last missing phase (4.3): resolve the release,
take the x86_64 linux-gnu server build and refuse to substitute another,
verify a pinned checksum if one was given, extract the binary - refusing
any archive entry that isn't a regular file, since a tarball is untrusted
input - and confirm the result reports the version its tag claimed.
Everything upstream of that last check is an assumption about someone
else's release process.

Two gates, separate on purpose. --yes is about intent. --recovery-point-
confirmed is a claim about the world: this tool cannot undo a migration
(4.8) and cannot check whether a snapshot exists, so a run that proceeded
without the operator asserting one would be proceeding on a hope.

Verified end to end against a real Stalwart 0.15.5 with email-style account
names, a named admin account, and seeded mail:

    MIGRATION COMPLETE. Mail was down for 6s.

Every cutover step green, including recalculate-quotas ("rebuilt disk
quotas for 2 account(s)") - the first time the x:Task wire format inferred
from Stalwart's schema reference has actually been exercised. It works,
now that endpoint discovery and role restoration make it reachable. After
the migration the named admin still administers, alice logs in with
unchanged credentials to the same four messages, and new SMTP delivery is
accepted.

Both refusal gates were tested, as was the failure path: an apply that
fails leaves the run stopped with the store part-migrated, and the error
says to restore the recovery point rather than restart the old version
against it.

internal/stage/doc.go

@@ -0,0 +1,6 @@
+// SPDX-FileCopyrightText: 2026 LINUXexpert-org
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+// Package stage fetches and verifies the target Stalwart binary, installing it alongside the running one.
+// See ARCHITECTURE.md §4.3 for the design.
+package stage

internal/stage/stage.go

@@ -0,0 +1,227 @@
+// SPDX-FileCopyrightText: 2026 LINUXexpert-org
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package stage
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/LINUXexpert-org/stalwart-migrator/internal/checkpoint"
+	"github.com/LINUXexpert-org/stalwart-migrator/internal/preflight"
+)
+
+// assetSuffix is the release asset holding a plain Linux x86_64 server
+// binary. Stalwart publishes several builds per release; this deliberately
+// matches only the one, rather than taking the first thing that looks
+// close - picking the FoundationDB build or a musl variant by accident is
+// the kind of mistake that surfaces as a puzzling runtime failure much
+// later.
+const assetSuffix = "stalwart-x86_64-unknown-linux-gnu.tar.gz"
+
+// binaryNameInArchive is the file to extract from that tarball.
+const binaryNameInArchive = "stalwart"
+
+// maxBinaryBytes caps extraction. The 0.16.14 server binary is ~100 MB; a
+// limit an order of magnitude above that stops a malformed or hostile
+// archive from filling the disk while leaving ample room for growth.
+const maxBinaryBytes = 1 << 30
+
+// Options configures staging.
+type Options struct {
+	// TargetVersion is the release to fetch ("0.16.14", or "latest").
+	TargetVersion string
+	// DestPath is where the extracted binary is written. It must not be
+	// the running binary's path: staging installs *alongside*, and cutover
+	// is what moves it into place.
+	DestPath string
+	// SHA256, when set, is the expected checksum of the downloaded archive.
+	// Recommended: the release process does not always publish a checksum
+	// manifest, so pinning is how a second run gets the guarantee the
+	// first one couldn't have.
+	SHA256     string
+	HTTPClient *http.Client
+}
+
+// Run downloads the target release, verifies what it can, extracts the
+// server binary to DestPath, and confirms the binary reports the version
+// that was asked for.
+//
+// That last check is the point of the phase. Everything upstream of it -
+// the tag lookup, the asset name, the archive layout - is an assumption
+// about someone else's release process, and the binary's own --version
+// output is the only thing that actually settles what was fetched. Cutover
+// checks it again before installing, deliberately: this phase and that one
+// can be separated by a long time and an operator's own file management.
+func Run(ctx context.Context, store *checkpoint.Store, rs *checkpoint.RunState, opts Options) (string, error) {
+	if opts.DestPath == "" {
+		return "", fmt.Errorf("stage: no destination path for the target binary")
+	}
+
+	outcome, err := store.RunStep(rs, checkpoint.PhaseStage, "stage-binary", func() (checkpoint.StepOutcome, error) {
+		release, err := preflight.ResolveRelease(ctx, opts.HTTPClient, opts.TargetVersion)
+		if err != nil {
+			return checkpoint.StepOutcome{}, err
+		}
+		asset := serverAsset(release)
+		if asset == nil {
+			names := make([]string, 0, len(release.Assets))
+			for _, a := range release.Assets {
+				names = append(names, a.Name)
+			}
+			return checkpoint.StepOutcome{}, fmt.Errorf(
+				"stage: release %s publishes no %s asset (found: %s) - this tool stages a Linux x86_64 server build and won't substitute another",
+				release.TagName, assetSuffix, strings.Join(names, ", "))
+		}
+
+		archivePath := opts.DestPath + ".tar.gz"
+		sum, err := download(ctx, opts.HTTPClient, asset.DownloadURL, archivePath)
+		if err != nil {
+			return checkpoint.StepOutcome{}, err
+		}
+		defer os.Remove(archivePath)
+
+		if opts.SHA256 != "" && !strings.EqualFold(sum, opts.SHA256) {
+			return checkpoint.StepOutcome{}, fmt.Errorf(
+				"stage: downloaded %s has sha256 %s but %s was expected - refusing to stage a binary that isn't the one that was pinned",
+				asset.Name, sum, opts.SHA256)
+		}
+
+		if err := extractBinary(archivePath, opts.DestPath); err != nil {
+			return checkpoint.StepOutcome{}, err
+		}
+
+		got, err := preflight.DetectVersion(ctx, opts.DestPath)
+		if err != nil {
+			return checkpoint.StepOutcome{}, fmt.Errorf("stage: staged binary at %s won't report its version: %w", opts.DestPath, err)
+		}
+		wanted := strings.TrimPrefix(release.TagName, "v")
+		if got != wanted {
+			return checkpoint.StepOutcome{}, fmt.Errorf(
+				"stage: staged binary reports %s but release %s was fetched - the release asset does not contain what its tag claims",
+				got, release.TagName)
+		}
+
+		pinNote := ""
+		if opts.SHA256 == "" {
+			pinNote = fmt.Sprintf("; no checksum was pinned - record sha256 %s to pin this download for future runs", sum)
+		}
+		return checkpoint.StepOutcome{
+			Detail: fmt.Sprintf("staged %s at %s, which reports %s%s", asset.Name, opts.DestPath, got, pinNote),
+			Extra:  got,
+		}, nil
+	})
+	if err != nil {
+		return "", err
+	}
+	_ = outcome
+	return opts.DestPath, nil
+}
+
+func serverAsset(release *preflight.Release) *preflight.ReleaseAsset {
+	for i := range release.Assets {
+		if release.Assets[i].Name == assetSuffix {
+			return &release.Assets[i]
+		}
+	}
+	return nil
+}
+
+// download fetches url to path and returns its SHA256.
+func download(ctx context.Context, client *http.Client, url, path string) (string, error) {
+	if client == nil {
+		client = &http.Client{}
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return "", err
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("stage: download %s: %w", url, err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("stage: download %s returned %s", url, resp.Status)
+	}
+	if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
+		return "", fmt.Errorf("stage: create %s: %w", filepath.Dir(path), err)
+	}
+	f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o640)
+	if err != nil {
+		return "", fmt.Errorf("stage: create %s: %w", path, err)
+	}
+	defer f.Close()
+	h := sha256.New()
+	if _, err := io.Copy(io.MultiWriter(f, h), io.LimitReader(resp.Body, maxBinaryBytes)); err != nil {
+		return "", fmt.Errorf("stage: write %s: %w", path, err)
+	}
+	if err := f.Sync(); err != nil {
+		return "", fmt.Errorf("stage: sync %s: %w", path, err)
+	}
+	return hex.EncodeToString(h.Sum(nil)), nil
+}
+
+// extractBinary pulls the server binary out of a release tarball.
+//
+// It searches by base name rather than by a fixed path, because the archive
+// layout is the release process's business and has already varied between
+// products (the separately-versioned CLI ships its binary a directory
+// down). Anything that isn't a regular file is refused rather than
+// followed: a tarball is untrusted input, and an entry that is a symlink or
+// carries a path escaping the destination has no legitimate reason to be
+// there.
+func extractBinary(archivePath, destPath string) error {
+	f, err := os.Open(archivePath)
+	if err != nil {
+		return fmt.Errorf("stage: open %s: %w", archivePath, err)
+	}
+	defer f.Close()
+	gz, err := gzip.NewReader(f)
+	if err != nil {
+		return fmt.Errorf("stage: %s is not gzip: %w", archivePath, err)
+	}
+	defer gz.Close()
+
+	tr := tar.NewReader(gz)
+	for {
+		header, err := tr.Next()
+		if err == io.EOF {
+			return fmt.Errorf("stage: %s contains no %q entry", archivePath, binaryNameInArchive)
+		}
+		if err != nil {
+			return fmt.Errorf("stage: read %s: %w", archivePath, err)
+		}
+		if filepath.Base(header.Name) != binaryNameInArchive {
+			continue
+		}
+		if header.Typeflag != tar.TypeReg {
+			return fmt.Errorf("stage: %q in %s is not a regular file (type %q) - refusing to follow it",
+				header.Name, archivePath, string(header.Typeflag))
+		}
+		out, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o755)
+		if err != nil {
+			return fmt.Errorf("stage: create %s: %w", destPath, err)
+		}
+		defer out.Close()
+		if _, err := io.Copy(out, io.LimitReader(tr, maxBinaryBytes)); err != nil {
+			return fmt.Errorf("stage: extract to %s: %w", destPath, err)
+		}
+		if err := out.Sync(); err != nil {
+			return fmt.Errorf("stage: sync %s: %w", destPath, err)
+		}
+		return nil
+	}
+}
+
+func releaseAPIBase() string        { return preflight.ReleaseAPIBase() }
+func setReleaseAPIBase(base string) { preflight.SetReleaseAPIBase(base) }

internal/stage/stage_test.go

@@ -0,0 +1,254 @@
+// SPDX-FileCopyrightText: 2026 LINUXexpert-org
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package stage
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/LINUXexpert-org/stalwart-migrator/internal/checkpoint"
+)
+
+// tarGzWith builds a release-shaped archive containing one entry.
+func tarGzWith(t *testing.T, name, body string, typeflag byte) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	gz := gzip.NewWriter(&buf)
+	tw := tar.NewWriter(gz)
+	hdr := &tar.Header{Name: name, Mode: 0o755, Size: int64(len(body)), Typeflag: typeflag}
+	if typeflag == tar.TypeSymlink {
+		hdr.Size = 0
+		hdr.Linkname = "/etc/passwd"
+	}
+	if err := tw.WriteHeader(hdr); err != nil {
+		t.Fatal(err)
+	}
+	if typeflag == tar.TypeReg {
+		if _, err := tw.Write([]byte(body)); err != nil {
+			t.Fatal(err)
+		}
+	}
+	tw.Close()
+	gz.Close()
+	return buf.Bytes()
+}
+
+// releaseServer stands in for the GitHub release API plus asset hosting.
+func releaseServer(t *testing.T, tag string, archive []byte, assetName string) *httptest.Server {
+	t.Helper()
+	var srv *httptest.Server
+	srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.HasSuffix(r.URL.Path, "/download") {
+			w.Write(archive)
+			return
+		}
+		json.NewEncoder(w).Encode(map[string]any{
+			"tag_name": tag,
+			"assets": []map[string]any{
+				{"name": "stalwart-foundationdb-x86_64-unknown-linux-gnu.tar.gz", "browser_download_url": srv.URL + "/wrong/download"},
+				{"name": assetName, "browser_download_url": srv.URL + "/right/download"},
+			},
+		})
+	}))
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+// A fake "binary" that reports a version, so the staged-version check has
+// something real to run.
+func versionScript(v string) string {
+	return fmt.Sprintf("#!/bin/sh\necho 'stalwart %s'\n", v)
+}
+
+func newRun(t *testing.T) (*checkpoint.Store, *checkpoint.RunState) {
+	t.Helper()
+	store := checkpoint.NewStore(filepath.Join(t.TempDir(), "runs"))
+	rs, err := store.Create("0.15.5", "0.16.14")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return store, rs
+}
+
+func withReleaseAPI(t *testing.T, srv *httptest.Server) {
+	t.Helper()
+	// preflight.ResolveRelease reads its base from a package var; point it
+	// at the fake for the duration of the test.
+	old := releaseAPIBase()
+	setReleaseAPIBase(srv.URL)
+	t.Cleanup(func() { setReleaseAPIBase(old) })
+}
+
+func TestRunStagesTheServerBuildAndVerifiesItsVersion(t *testing.T) {
+	archive := tarGzWith(t, "stalwart", versionScript("0.16.14"), tar.TypeReg)
+	srv := releaseServer(t, "v0.16.14", archive, assetSuffix)
+	withReleaseAPI(t, srv)
+	store, rs := newRun(t)
+	dest := filepath.Join(t.TempDir(), "stalwart-0.16.14")
+
+	path, err := Run(context.Background(), store, rs, Options{
+		TargetVersion: "0.16.14", DestPath: dest, HTTPClient: srv.Client(),
+	})
+	if err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+	if path != dest {
+		t.Errorf("path = %q, want %q", path, dest)
+	}
+	info, err := os.Stat(dest)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if info.Mode().Perm()&0o111 == 0 {
+		t.Errorf("staged binary mode = %v, want executable", info.Mode().Perm())
+	}
+	// The archive it must NOT have taken is the FoundationDB build.
+	if _, err := os.Stat(dest + ".tar.gz"); !os.IsNotExist(err) {
+		t.Error("the downloaded archive should be cleaned up after extraction")
+	}
+}
+
+// The release publishes several builds; taking the first plausible one
+// would stage a FoundationDB or musl variant and surface as a puzzling
+// runtime failure much later.
+func TestRunRefusesWhenTheServerBuildIsAbsent(t *testing.T) {
+	archive := tarGzWith(t, "stalwart", versionScript("0.16.14"), tar.TypeReg)
+	srv := releaseServer(t, "v0.16.14", archive, "stalwart-aarch64-unknown-linux-gnu.tar.gz")
+	withReleaseAPI(t, srv)
+	store, rs := newRun(t)
+
+	_, err := Run(context.Background(), store, rs, Options{
+		TargetVersion: "0.16.14", DestPath: filepath.Join(t.TempDir(), "s"), HTTPClient: srv.Client(),
+	})
+	if err == nil {
+		t.Fatal("want a refusal when the x86_64 server build isn't published")
+	}
+	if !strings.Contains(err.Error(), "won't substitute another") {
+		t.Errorf("error %q should say it refuses to substitute a different build", err)
+	}
+}
+
+// The binary's own --version is the only thing that settles what was
+// actually fetched; everything upstream is an assumption about someone
+// else's release process.
+func TestRunRefusesAnArchiveThatDoesNotMatchItsTag(t *testing.T) {
+	archive := tarGzWith(t, "stalwart", versionScript("0.16.9"), tar.TypeReg)
+	srv := releaseServer(t, "v0.16.14", archive, assetSuffix)
+	withReleaseAPI(t, srv)
+	store, rs := newRun(t)
+
+	_, err := Run(context.Background(), store, rs, Options{
+		TargetVersion: "0.16.14", DestPath: filepath.Join(t.TempDir(), "s"), HTTPClient: srv.Client(),
+	})
+	if err == nil {
+		t.Fatal("want a refusal when the asset contains a different version than its tag claims")
+	}
+	if !strings.Contains(err.Error(), "0.16.9") {
+		t.Errorf("error %q should report what the binary actually said", err)
+	}
+}
+
+func TestRunHonoursAPinnedChecksum(t *testing.T) {
+	archive := tarGzWith(t, "stalwart", versionScript("0.16.14"), tar.TypeReg)
+	srv := releaseServer(t, "v0.16.14", archive, assetSuffix)
+	withReleaseAPI(t, srv)
+	store, rs := newRun(t)
+
+	_, err := Run(context.Background(), store, rs, Options{
+		TargetVersion: "0.16.14", DestPath: filepath.Join(t.TempDir(), "s"),
+		SHA256: "0000000000000000000000000000000000000000000000000000000000000000", HTTPClient: srv.Client(),
+	})
+	if err == nil {
+		t.Fatal("want a refusal when the download doesn't match the pin")
+	}
+	if !strings.Contains(err.Error(), "was pinned") {
+		t.Errorf("error %q should say the pin was violated", err)
+	}
+
+	// And the matching pin is accepted.
+	sum := sha256.Sum256(archive)
+	store2, rs2 := newRun(t)
+	if _, err := Run(context.Background(), store2, rs2, Options{
+		TargetVersion: "0.16.14", DestPath: filepath.Join(t.TempDir(), "s"),
+		SHA256: hex.EncodeToString(sum[:]), HTTPClient: srv.Client(),
+	}); err != nil {
+		t.Fatalf("a matching pin should be accepted: %v", err)
+	}
+}
+
+// A tarball is untrusted input.
+func TestExtractRefusesANonRegularEntry(t *testing.T) {
+	dir := t.TempDir()
+	archivePath := filepath.Join(dir, "a.tar.gz")
+	if err := os.WriteFile(archivePath, tarGzWith(t, "stalwart", "", tar.TypeSymlink), 0o640); err != nil {
+		t.Fatal(err)
+	}
+	err := extractBinary(archivePath, filepath.Join(dir, "out"))
+	if err == nil {
+		t.Fatal("want a refusal for a symlink entry, got nil")
+	}
+	if !strings.Contains(err.Error(), "refusing to follow") {
+		t.Errorf("error %q should say it refuses to follow it", err)
+	}
+}
+
+// The separately-versioned CLI ships its binary a directory down, so
+// searching by base name rather than exact path is deliberate.
+func TestExtractFindsTheBinaryInASubdirectory(t *testing.T) {
+	dir := t.TempDir()
+	archivePath := filepath.Join(dir, "a.tar.gz")
+	if err := os.WriteFile(archivePath, tarGzWith(t, "stalwart-x86_64/stalwart", versionScript("0.16.14"), tar.TypeReg), 0o640); err != nil {
+		t.Fatal(err)
+	}
+	dest := filepath.Join(dir, "out")
+	if err := extractBinary(archivePath, dest); err != nil {
+		t.Fatalf("extractBinary: %v", err)
+	}
+	if body, _ := os.ReadFile(dest); !strings.Contains(string(body), "0.16.14") {
+		t.Errorf("extracted %q, want the binary from the subdirectory", body)
+	}
+}
+
+// Re-running a completed stage must not re-download.
+func TestRunIsSkippedOnResume(t *testing.T) {
+	archive := tarGzWith(t, "stalwart", versionScript("0.16.14"), tar.TypeReg)
+	var downloads int
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.HasSuffix(r.URL.Path, "/download") {
+			downloads++
+			w.Write(archive)
+			return
+		}
+		json.NewEncoder(w).Encode(map[string]any{
+			"tag_name": "v0.16.14",
+			"assets":   []map[string]any{{"name": assetSuffix, "browser_download_url": "http://" + r.Host + "/right/download"}},
+		})
+	}))
+	defer srv.Close()
+	withReleaseAPI(t, srv)
+	store, rs := newRun(t)
+	opts := Options{TargetVersion: "0.16.14", DestPath: filepath.Join(t.TempDir(), "s"), HTTPClient: srv.Client()}
+
+	if _, err := Run(context.Background(), store, rs, opts); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := Run(context.Background(), store, rs, opts); err != nil {
+		t.Fatal(err)
+	}
+	if downloads != 1 {
+		t.Errorf("downloaded %d time(s), want 1 - a resumed run must not re-fetch 100 MB", downloads)
+	}
+}