Replace the sandbox dry run with a read-only rehearsal
`run --dry-run` cloned the data directory into a sandbox, migrated the copy,
booted it, and compared content before and after. Running that design
against a real 0.15.5 instance and a real production settings corpus
retired it:
* The mechanics were never the risk. Backup, dump, convert and the
recovery-mode store migration all worked essentially first time.
* Its final comparison cannot work at all. It needs the migrated sandbox
to answer an API, and server.listener is not among the settings
migrate_v016.py carries - so a migrated instance has no listeners and
answers on nothing. That is the true post-migration state, not a
sandbox artifact to engineer around.
* The expensive half bought the least: against a 3.6 GB production store
it copies the data twice, reading a live mail store, to prove RocksDB
files copy and recovery mode can open them.
Meanwhile the cheap half found every problem that would have derailed a
real migration - an empty defaultHostname v0.16 rejects, passwords v0.16
refuses to create, and a 12,182-key reconstruction worklist - and needs no
data copy at all.
So `stalwart-migrate rehearse`: preflight, dump, convert, report. It copies
nothing, starts no server, and never writes to the store, so it is safe to
run against production repeatedly without a maintenance window. It needs no
target binary either, since convert is pure Python.
The scratch directory is cleaned up as before, with the rehearsal's two
conclusions lifted out first and recorded as artifacts: export.json (what
will carry over) and unmigrated.txt (what will not). Recording an artifact
whose path was about to be deleted was a bug in the first cut of this;
both now resolve.
`run` keeps its refusal and explains where rehearse went. `--dry-run` is
kept as a flag purely to say what replaced it.
Verified against the smoke VM end to end: rehearsal completes read-only in
seconds and reports 3505 unmigrated settings on a default install,
listeners included.
This commit is contained in:
@@ -0,0 +1,246 @@
|
||||
// SPDX-FileCopyrightText: 2026 LINUXexpert-org
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/LINUXexpert-org/stalwart-migrator/internal/backup"
|
||||
"github.com/LINUXexpert-org/stalwart-migrator/internal/checkpoint"
|
||||
"github.com/LINUXexpert-org/stalwart-migrator/internal/plan"
|
||||
"github.com/LINUXexpert-org/stalwart-migrator/internal/preflight"
|
||||
)
|
||||
|
||||
// runRehearse implements `stalwart-migrate rehearse` (ARCHITECTURE.md §4.9):
|
||||
// preflight, dump this instance's settings and principals, convert them with
|
||||
// migrate_v016.py, and report both halves of the result - what will carry
|
||||
// over, and what will not.
|
||||
//
|
||||
// It copies no data, clones nothing, starts no server, and never writes to
|
||||
// the store, so it is safe to run against production repeatedly and without
|
||||
// a maintenance window. That is a deliberate narrowing from the sandbox-
|
||||
// cloning dry run this replaces: cloning the store proved only that the
|
||||
// store migrates and opens, at the cost of copying the data twice and
|
||||
// reading a live store to do it, while the half that found every real
|
||||
// problem - an empty defaultHostname v0.16 rejects, passwords v0.16 refuses,
|
||||
// and a 12,182-key reconstruction worklist - needs no copy at all.
|
||||
//
|
||||
// The worklist is the point. Measured against a real production instance,
|
||||
// migrate_v016.py carried 219 of 12,401 settings; server.listener was not
|
||||
// among them, so a migrated instance answers on no ports until an operator
|
||||
// rebuilds them. Anything that reported such a migration as a success
|
||||
// without saying so would be actively misleading.
|
||||
func runRehearse(args []string) (err error) {
|
||||
fs := flag.NewFlagSet("rehearse", flag.ExitOnError)
|
||||
binaryPath := fs.String("binary", "/usr/local/bin/stalwart", "path to the currently-installed stalwart binary")
|
||||
configPath := fs.String("config", "/etc/stalwart/config.toml", "path to stalwart's current config file")
|
||||
dataDir := fs.String("data-dir", "/var/lib/stalwart", "stalwart data directory (read-only here; used by preflight's checks)")
|
||||
containerName := fs.String("container", "stalwart", "docker container name, if applicable")
|
||||
adminURL := fs.String("admin-url", "", "base URL for the live instance's admin/JMAP API (required)")
|
||||
adminUser := fs.String("admin-user", "", "admin username")
|
||||
adminPassword := fs.String("admin-password", os.Getenv("STALWART_MIGRATE_ADMIN_PASSWORD"),
|
||||
"admin password (or set STALWART_MIGRATE_ADMIN_PASSWORD)")
|
||||
targetVersion := fs.String("target", "latest", `target Stalwart version, or "latest"`)
|
||||
stateDir := fs.String("state-dir", checkpoint.DefaultBaseDir, "directory to store run checkpoints in")
|
||||
workDir := fs.String("work-dir", "/var/lib/stalwart-migrator/work", "scratch directory for the dumps and converted plan (cleaned up afterward - see --keep-artifacts)")
|
||||
pythonPath := fs.String("python", "python3", "path to python3")
|
||||
migrationScriptSHA256 := fs.String("migration-script-sha256", "", "pinned sha256 of migrate_v016.py (recommended; the first run prints the hash to pin)")
|
||||
minFree := fs.Float64("min-free-multiple", 2.0, "free-space multiple preflight checks for; rehearsal itself copies nothing")
|
||||
keepArtifacts := fs.Bool("keep-artifacts", false, "don't delete work-dir/<run-id> afterward")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
if *adminURL == "" {
|
||||
return fmt.Errorf("--admin-url is required: rehearsal converts the settings this instance actually has, which means reading them from it")
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
httpClient := &http.Client{}
|
||||
|
||||
if err := os.MkdirAll(*workDir, 0o750); err != nil {
|
||||
return fmt.Errorf("create work dir %s: %w", *workDir, err)
|
||||
}
|
||||
|
||||
store := checkpoint.NewStore(*stateDir)
|
||||
rs, err := store.Create("", *targetVersion)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create run: %w", err)
|
||||
}
|
||||
fmt.Printf("run id: %s\n\n", rs.RunID)
|
||||
|
||||
runWorkDir := filepath.Join(*workDir, rs.RunID)
|
||||
runStateDir := filepath.Join(*stateDir, rs.RunID)
|
||||
// The rehearsal's two conclusions survive cleanup: the worklist of what
|
||||
// won't carry over, and the plan of what will. Everything else in the
|
||||
// work directory is scratch. Recording an artifact that points into a
|
||||
// directory about to be deleted would leave the checkpoint referring to
|
||||
// files that aren't there.
|
||||
keptWorklist := filepath.Join(runStateDir, "unmigrated.txt")
|
||||
keptPlan := filepath.Join(runStateDir, "export.json")
|
||||
defer func() {
|
||||
if _, statErr := os.Stat(runWorkDir); os.IsNotExist(statErr) {
|
||||
return
|
||||
}
|
||||
if *keepArtifacts {
|
||||
fmt.Printf("\nartifacts kept at %s (--keep-artifacts)\n", runWorkDir)
|
||||
return
|
||||
}
|
||||
if rmErr := os.RemoveAll(runWorkDir); rmErr != nil {
|
||||
fmt.Fprintf(os.Stderr, "\nwarning: failed to clean up %s: %v (remove it manually)\n", runWorkDir, rmErr)
|
||||
return
|
||||
}
|
||||
fmt.Printf("\ncleaned up %s; the run log is at %s\n", runWorkDir, filepath.Join(runStateDir, "state.json"))
|
||||
}()
|
||||
|
||||
fmt.Println("--- preflight ---")
|
||||
checker := preflight.New(preflight.Options{
|
||||
BinaryPath: *binaryPath, ConfigPath: *configPath, DataDir: *dataDir, ContainerName: *containerName,
|
||||
AdminURL: *adminURL, AdminUser: *adminUser, AdminPassword: *adminPassword,
|
||||
TargetVersion: *targetVersion, MinFreeMultiple: *minFree, HTTPClient: httpClient,
|
||||
})
|
||||
pfReport, err := checker.Run(ctx, store, rs)
|
||||
fmt.Print(pfReport.String())
|
||||
if err != nil {
|
||||
return fmt.Errorf("preflight failed to complete: %w", err)
|
||||
}
|
||||
if pfReport.Blocking() {
|
||||
return fmt.Errorf("preflight found blocking issues - see FAIL lines above")
|
||||
}
|
||||
|
||||
p, err := plan.Decide(rs.SourceVersion, rs.TargetVersion)
|
||||
if err != nil {
|
||||
return fmt.Errorf("plan: %w", err)
|
||||
}
|
||||
fmt.Printf("\nplan: %s\n", p.Reason)
|
||||
if !p.CrossesMajorBoundary {
|
||||
fmt.Println("\nthis is a same-boundary patch upgrade: its settings don't need converting, " +
|
||||
"so there is nothing to rehearse. A real run would be a binary swap and restart.")
|
||||
return nil
|
||||
}
|
||||
|
||||
scriptDest := filepath.Join(runWorkDir, "migrate_v016.py")
|
||||
settingsPath := filepath.Join(runWorkDir, "settings.json")
|
||||
principalsPath := filepath.Join(runWorkDir, "principals.json")
|
||||
convertedConfig := filepath.Join(runWorkDir, "config.json")
|
||||
convertedExport := filepath.Join(runWorkDir, "export.json")
|
||||
unmigratedPath := filepath.Join(runWorkDir, "unmigrated.txt")
|
||||
|
||||
fmt.Println("\n--- dump (read-only) ---")
|
||||
if _, err := store.RunStep(rs, checkpoint.PhaseBackup, "settings-dump", func() (checkpoint.StepOutcome, error) {
|
||||
if err := os.MkdirAll(runWorkDir, 0o750); err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
sum, err := backup.DownloadFile(ctx, httpClient, backup.DefaultMigrationScriptURL, scriptDest, *migrationScriptSHA256)
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
pinNote := ""
|
||||
if *migrationScriptSHA256 == "" {
|
||||
pinNote = fmt.Sprintf(" (no pin configured - record sha256 %s as --migration-script-sha256 to pin it)", sum)
|
||||
}
|
||||
if err := backup.RunSettingsDump(ctx, backup.SettingsDumpOptions{
|
||||
PythonPath: *pythonPath, ScriptPath: scriptDest, URL: *adminURL,
|
||||
Username: *adminUser, Password: *adminPassword,
|
||||
SettingsPath: settingsPath, PrincipalsPath: principalsPath,
|
||||
}); err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
_, settingsSize, err := backup.HashFile(settingsPath)
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
_, principalsSize, err := backup.HashFile(principalsPath)
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
return checkpoint.StepOutcome{
|
||||
Detail: fmt.Sprintf("dumped settings (%d bytes) and principals (%d bytes) from %s%s",
|
||||
settingsSize, principalsSize, *adminURL, pinNote),
|
||||
}, nil
|
||||
}); err != nil {
|
||||
return fmt.Errorf("settings dump: %w", err)
|
||||
}
|
||||
fmt.Println(rs.Outcome(checkpoint.PhaseBackup, "settings-dump").Detail)
|
||||
|
||||
fmt.Println("\n--- convert ---")
|
||||
if _, err := store.RunStep(rs, checkpoint.PhaseStage, "convert-settings", func() (checkpoint.StepOutcome, error) {
|
||||
if err := backup.RunSettingsConvert(ctx, backup.SettingsConvertOptions{
|
||||
PythonPath: *pythonPath, ScriptPath: scriptDest,
|
||||
SettingsPath: settingsPath, PrincipalsPath: principalsPath,
|
||||
ConfigPath: convertedConfig, OutputPath: convertedExport,
|
||||
// migrate_v016.py writes unmigrated.txt into its working
|
||||
// directory; without this it lands wherever the operator
|
||||
// happened to be, or fails the convert if that isn't writable.
|
||||
WorkDir: runWorkDir,
|
||||
}); err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
detail := "converted this instance's settings into a v0.16 apply plan"
|
||||
if report, readErr := backup.ReadUnmigratedReport(unmigratedPath); readErr == nil && report != nil {
|
||||
detail += fmt.Sprintf("; %d setting(s) will NOT carry over", report.TotalKeys)
|
||||
}
|
||||
return checkpoint.StepOutcome{Detail: detail}, nil
|
||||
}); err != nil {
|
||||
return fmt.Errorf("convert settings: %w", err)
|
||||
}
|
||||
|
||||
if err := copyFile(convertedExport, keptPlan); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "warning: couldn't preserve the apply plan: %v\n", err)
|
||||
} else if sum, size, hashErr := backup.HashFile(keptPlan); hashErr == nil {
|
||||
rs.RecordArtifact("converted-export", checkpoint.Artifact{Path: keptPlan, SHA256: sum, SizeBytes: size})
|
||||
fmt.Printf("apply plan: %s (%d bytes) - what WILL carry over\n", keptPlan, size)
|
||||
}
|
||||
|
||||
unmigrated, err := backup.ReadUnmigratedReport(unmigratedPath)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "warning: couldn't read the unmigrated-settings report: %v\n", err)
|
||||
} else if unmigrated != nil && unmigrated.TotalKeys > 0 {
|
||||
if err := copyFile(unmigratedPath, keptWorklist); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "warning: couldn't preserve the worklist: %v\n", err)
|
||||
} else {
|
||||
unmigrated.Path = keptWorklist
|
||||
if sum, size, hashErr := backup.HashFile(keptWorklist); hashErr == nil {
|
||||
rs.RecordArtifact("unmigrated-settings", checkpoint.Artifact{Path: keptWorklist, SHA256: sum, SizeBytes: size})
|
||||
}
|
||||
}
|
||||
fmt.Printf("\n !! %s\n", unmigrated.Summary(10))
|
||||
fmt.Println(" These do not carry over. Rebuild them on the migrated instance before it serves mail -")
|
||||
fmt.Println(" note that server.listener is typically among them, so until you do, it answers on nothing.")
|
||||
}
|
||||
if err := store.Save(rs); err != nil {
|
||||
return fmt.Errorf("save run state: %w", err)
|
||||
}
|
||||
|
||||
fmt.Printf("\nREHEARSAL COMPLETE for run %s. Nothing was modified: no data was copied, no server was started,\n"+
|
||||
"and the store was never written to.\n", rs.RunID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// copyFile duplicates src to dst, used to lift the worklist out of the
|
||||
// scratch directory before it's cleaned up.
|
||||
func copyFile(src, dst string) error {
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0o750); err != nil {
|
||||
return err
|
||||
}
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o640)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close()
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
return err
|
||||
}
|
||||
return out.Sync()
|
||||
}
|
||||
Reference in New Issue
Block a user