From 0739866c148f8a5bee1fe4a573c6538e914979f7 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Mon, 24 Aug 2026 15:12:35 -0700 Subject: [PATCH] Let a host with no internet supply the migration script itself `run` and `rehearse` always fetched migrate_v016.py from GitHub, so a mail server with no route out could not be migrated at all - an air-gapped host, or a clone deliberately cut off so it cannot renew certificates or deliver queued mail for the domains it was copied from. --migration-script takes a local copy instead, still checked against --migration-script-sha256 when one is pinned. Found while staging a production clone for a dress rehearsal: the clone has no route out on purpose, and that is exactly the property that stops a copy of a live mail server doing something in the real world. --- cmd/stalwart-migrate/rehearse.go | 3 +- cmd/stalwart-migrate/run.go | 3 +- internal/backup/settingsdump.go | 26 ++++++++++++++ internal/backup/settingsdump_test.go | 54 ++++++++++++++++++++++++++++ 4 files changed, 84 insertions(+), 2 deletions(-) diff --git a/cmd/stalwart-migrate/rehearse.go b/cmd/stalwart-migrate/rehearse.go index abd364b..b2a69a3 100644 --- a/cmd/stalwart-migrate/rehearse.go +++ b/cmd/stalwart-migrate/rehearse.go @@ -55,6 +55,7 @@ func runRehearse(args []string) (err error) { stalwartCLI := fs.String("stalwart-cli", "stalwart-cli", "path to stalwart-cli (v1.0.2 or later; a separate download from the server) - rehearsal doesn't invoke it, but checks it so `run` doesn't fail after stopping the service") migrationScriptSHA256 := fs.String("migration-script-sha256", "", "pinned sha256 of migrate_v016.py (recommended; the first run prints the hash to pin)") + migrationScriptPath := fs.String("migration-script", "", "use a local copy of migrate_v016.py instead of fetching it (for a host with no route to the internet)") minFree := fs.Float64("min-free-multiple", 2.0, "free-space multiple preflight checks for; rehearsal itself copies nothing") keepArtifacts := fs.Bool("keep-artifacts", false, "don't delete work-dir/ afterward") if err := fs.Parse(args); err != nil { @@ -146,7 +147,7 @@ func runRehearse(args []string) (err error) { if err := os.MkdirAll(runWorkDir, 0o750); err != nil { return checkpoint.StepOutcome{}, err } - sum, err := backup.DownloadFile(ctx, httpClient, backup.DefaultMigrationScriptURL, scriptDest, *migrationScriptSHA256) + sum, err := backup.ProvideFile(ctx, httpClient, *migrationScriptPath, backup.DefaultMigrationScriptURL, scriptDest, *migrationScriptSHA256) if err != nil { return checkpoint.StepOutcome{}, err } diff --git a/cmd/stalwart-migrate/run.go b/cmd/stalwart-migrate/run.go index a5f310d..45eb743 100644 --- a/cmd/stalwart-migrate/run.go +++ b/cmd/stalwart-migrate/run.go @@ -66,6 +66,7 @@ func runRun(args []string) (err error) { pythonPath := fs.String("python", "python3", "path to python3") stalwartCLI := fs.String("stalwart-cli", "stalwart-cli", "path to stalwart-cli (v1.0.2 or later; a separate download from the server)") scriptSHA := fs.String("migration-script-sha256", "", "pinned sha256 of migrate_v016.py") + scriptPath := fs.String("migration-script", "", "use a local copy of migrate_v016.py instead of fetching it (for a host with no route to the internet)") binarySHA := fs.String("target-binary-sha256", "", "pinned sha256 of the target release archive") minFree := fs.Float64("min-free-multiple", 2.0, "required free disk space as a multiple of the data directory size") recalcQuotas := fs.Bool("recalculate-quotas", true, "schedule the post-migration quota rebuild") @@ -196,7 +197,7 @@ func runRun(args []string) (err error) { if p.CrossesMajorBoundary { fmt.Println("\n--- dump (service still up) ---") if _, err := store.RunStep(rs, checkpoint.PhaseBackup, "settings-dump", func() (checkpoint.StepOutcome, error) { - if _, err := backup.DownloadFile(ctx, httpClient, backup.DefaultMigrationScriptURL, script, *scriptSHA); err != nil { + if _, err := backup.ProvideFile(ctx, httpClient, *scriptPath, backup.DefaultMigrationScriptURL, script, *scriptSHA); err != nil { return checkpoint.StepOutcome{}, err } if err := backup.RunSettingsDump(ctx, backup.SettingsDumpOptions{ diff --git a/internal/backup/settingsdump.go b/internal/backup/settingsdump.go index dabe0ef..19f8489 100644 --- a/internal/backup/settingsdump.go +++ b/internal/backup/settingsdump.go @@ -26,6 +26,32 @@ import ( // pinning discussion on DownloadFile and ARCHITECTURE.md ยง8. const DefaultMigrationScriptURL = "https://raw.githubusercontent.com/stalwartlabs/stalwart/main/resources/scripts/migrate_v016.py" +// ProvideFile puts the migration script at destPath, from srcPath when one +// is given and from url otherwise, and returns its SHA256. +// +// A local copy is not only a convenience for testing. A mail server with no +// route to the internet - an air-gapped host, or a clone deliberately cut +// off so it cannot renew certificates or deliver queued mail for the domains +// it was copied from - cannot fetch anything, and could not be migrated at +// all without this. +func ProvideFile(ctx context.Context, httpClient *http.Client, srcPath, url, destPath, expectedSHA256 string) (sha256Hex string, err error) { + if srcPath == "" { + return DownloadFile(ctx, httpClient, url, destPath, expectedSHA256) + } + data, err := os.ReadFile(srcPath) + if err != nil { + return "", fmt.Errorf("backup: read %s: %w", srcPath, err) + } + sum := fmt.Sprintf("%x", sha256.Sum256(data)) + if expectedSHA256 != "" && !strings.EqualFold(sum, expectedSHA256) { + return "", fmt.Errorf("backup: %s has sha256 %s, expected %s", srcPath, sum, expectedSHA256) + } + if err := os.WriteFile(destPath, data, 0o640); err != nil { + return "", fmt.Errorf("backup: write %s: %w", destPath, err) + } + return sum, nil +} + // DownloadFile fetches url to destPath and returns its SHA256. If // expectedSHA256 is non-empty, a mismatching download is rejected (and the // partial file removed) - this is how a pinned migration-script hash is diff --git a/internal/backup/settingsdump_test.go b/internal/backup/settingsdump_test.go index 005baac..6578486 100644 --- a/internal/backup/settingsdump_test.go +++ b/internal/backup/settingsdump_test.go @@ -7,6 +7,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "fmt" "net/http" "net/http/httptest" "os" @@ -377,3 +378,56 @@ func TestClassifyUnknownPrefixesDefaultToReview(t *testing.T) { t.Errorf("note = %q, want empty for an unclassified prefix", note) } } + +// A host with no route to the internet - an air-gapped server, or a clone cut +// off so it cannot renew certificates or deliver queued mail for the domains +// it was copied from - cannot fetch the migration script, and without a local +// copy could not be migrated at all. +func TestProvideFileUsesALocalCopy(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "migrate_v016.py") + if err := os.WriteFile(src, []byte("print('hello')\n"), 0o644); err != nil { + t.Fatal(err) + } + dest := filepath.Join(dir, "out.py") + + // A client that would fail loudly if it were used. + refuse := &http.Client{Transport: roundTripperFunc(func(*http.Request) (*http.Response, error) { + return nil, fmt.Errorf("network must not be touched when a local copy was given") + })} + + sum, err := ProvideFile(context.Background(), refuse, src, DefaultMigrationScriptURL, dest, "") + if err != nil { + t.Fatalf("ProvideFile: %v", err) + } + got, err := os.ReadFile(dest) + if err != nil || string(got) != "print('hello')\n" { + t.Fatalf("dest = %q, %v", got, err) + } + if want := fmt.Sprintf("%x", sha256.Sum256([]byte("print('hello')\n"))); sum != want { + t.Fatalf("sha256 = %s, want %s", sum, want) + } +} + +func TestProvideFileChecksThePinnedHash(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "migrate_v016.py") + if err := os.WriteFile(src, []byte("print('hello')\n"), 0o644); err != nil { + t.Fatal(err) + } + _, err := ProvideFile(context.Background(), nil, src, DefaultMigrationScriptURL, filepath.Join(dir, "out.py"), "deadbeef") + if err == nil { + t.Fatal("a local copy must still be checked against a pinned hash") + } +} + +func TestProvideFileReportsAMissingLocalCopy(t *testing.T) { + dir := t.TempDir() + if _, err := ProvideFile(context.Background(), nil, filepath.Join(dir, "nope.py"), DefaultMigrationScriptURL, filepath.Join(dir, "out.py"), ""); err == nil { + t.Fatal("expected an error for a local copy that is not there") + } +} + +type roundTripperFunc func(*http.Request) (*http.Response, error) + +func (f roundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }